00000800:00000200:3.1F:1769073240.780124:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [0000000021a1164d] -> 12345-192.168.204.160@tcp (2) 00000800:00000200:2.1F:1769073240.801009:0:2393:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:2.1:1769073240.801031:0:2393:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.160@tcp ip 192.168.204.160:988 00000800:00000200:2.1:1769073240.801037:0:2393:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000cd86b3e4 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:0.1F:1769073240.828304:0:2393:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:3.1:1769073240.845476:0:2393:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00020000:00200000:0.0:1769074780.097411:0:63007:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:0.0:1769074780.097417:0:63007:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 30 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074780.097445:0:63007:0:(rw.c:2016:ll_readpage()) fast read pgno: 0 00000080:00200000:0.0:1769074780.097467:0:63007:0:(file.c:1994:ll_file_io_generic()) f200b.sanity-flr: read ppos: 0, bytes: 4194304 00000080:00200000:0.0:1769074780.097473:0:63007:0:(file.c:1759:ll_io_set_mirror()) f200b.sanity-flr: desiginated mirror: 0 00000020:00200000:0.0:1769074780.097481:0:63007:0:(cl_io.c:214:cl_io_rw_init()) header@0000000006fe93a2[0x4, 1, [0x200000405:0x1fb:0x0] hash] 00000020:00200000:0.0:1769074780.097483:0:63007:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 4194304) 0 0 00000080:00200000:0.0:1769074780.097486:0:63007:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 30 restore needed 0 00020000:00200000:0.0:1769074780.097495:0:63007:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074780.097497:0:63007:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:0.0:1769074780.097504:0:63007:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 4194304, 1048576 00020000:00200000:0.0:1769074780.097507:0:63007:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074780.097523:0:63007:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:0.0:1769074780.097538:0:63007:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074780.097543:0:63007:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568dffb0, osc lock ffff9b0345a6c850, flags 0x400000000000000 00010000:00010000:0.0:1769074780.097569:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.097585:0:63007:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.097740:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.097754:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.097762:0:63007:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.097772:0:63007:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [0, 1048576) 00000080:00200000:0.0:1769074780.097787:0:63007:0:(vvp_io.c:886:vvp_io_read_start()) count:1048576 iocb pos:0 00010000:00010000:0.0:1769074780.097845:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 4/2,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.097856:0:63007:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (4096 8191) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 4/2,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.100007:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4200/0xb7d6bf619879c194 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54392 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.100032:0:63007:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 1052671) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4200/0xb7d6bf619879c194 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54392 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.109713:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4400/0xb7d6bf619879c19b lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543b5 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.109743:0:63007:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 1052671) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4400/0xb7d6bf619879c19b lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543b5 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.114968:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c1a2 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a0 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.115007:0:63007:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 1052671) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c1a2 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a0 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.118106:0:63007:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff9b03515b93c0 00000008:00010000:0.0:1769074780.118305:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b93c0 00010000:00010000:0.0:1769074780.118351:0:28211:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074780.118434:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a39500 - object 0x2c0000401:700 offset 0<>1048576 00000008:00010000:0.0:1769074780.118484:0:63007:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff9b03515b9f00 00000008:00010000:0.0:1769074780.118601:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9f00 00010000:00010000:0.0:1769074780.118624:0:28211:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 1052671) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4200/0xb7d6bf619879c194 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54392 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074780.118671:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a38700 - object 0x280000401:698 offset 1048576<>2097152 00000008:00010000:0.0:1769074780.118688:0:63007:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff9b03515b9a80 00000008:00010000:0.0:1769074780.118954:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9a80 00010000:00010000:0.0:1769074780.118987:0:28211:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 1052671) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4400/0xb7d6bf619879c19b lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543b5 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074780.119036:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a3ad80 - object 0x280000401:699 offset 1048576<>2097152 00000008:00010000:0.0:1769074780.119054:0:63007:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff9b03515b9180 00010000:00010000:0.0:1769074780.119065:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.119074:0:63007:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.119083:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4200/0xb7d6bf619879c194 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54392 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.119090:0:63007:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4200/0xb7d6bf619879c194 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54392 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.119109:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4400/0xb7d6bf619879c19b lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543b5 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.119115:0:63007:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4400/0xb7d6bf619879c19b lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543b5 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.119121:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c1a2 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543a0 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.119127:0:63007:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c1a2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a0 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.119142:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9180 00010000:00010000:0.0:1769074780.119161:0:28211:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 1052671) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c1a2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a0 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074780.119202:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a3b480 - object 0x2c0000401:702 offset 1048576<>2097152 00000100:00100000:0.0:1769074780.119493:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616921600:192.168.204.160@tcp:3:cat.0 00000100:00100000:0.0:1769074780.119614:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616921856:192.168.204.160@tcp:3:cat.0 00000100:00100000:0.0:1769074780.119659:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074780.119666:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616921728:192.168.204.160@tcp:3:cat.0 00000100:00100000:0.0:1769074780.119694:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616921984:192.168.204.160@tcp:3:cat.0 00000080:00200000:0.0:1769074780.130927:0:63001:0:(rw26.c:912:ll_write_end()) pos 4633600, len 3072, copied 3072 00000080:00200000:0.0:1769074780.130940:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000bf218819[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.130944:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000320b96da 17ffffc0001029 2:0 ffff9b035da49e58 1131 lru 00000080:00200000:0.0:1769074780.130950:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000bce03e13 619: 1< 1 - - > 2< 2535424 0 4096 0x0 0x40009 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.130953:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000bf218819 00000080:00200000:0.0:1769074780.130954:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 1. 00000080:00200000:0.0:1769074780.130959:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1132 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131024:0:63001:0:(rw26.c:912:ll_write_end()) pos 4636672, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131030:0:63001:0:(rw26.c:969:ll_write_end()) page@000000002e6a91dd[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131033:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000009f2b5d39 17ffffc0001001 3:0 ffff9b03439e6bd0 1132 lru 00000080:00200000:0.0:1769074780.131038:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000342f5f55 620: 1< 0 - - > 2< 2539520 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131039:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000002e6a91dd 00000080:00200000:0.0:1769074780.131041:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 2. 00000080:00200000:0.0:1769074780.131043:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1133 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131063:0:63001:0:(rw26.c:912:ll_write_end()) pos 4640768, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131067:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000a4218c38[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131070:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000a0d3d783 17ffffc0001021 2:0 ffff9b03439e6870 1133 lru 00000080:00200000:0.0:1769074780.131075:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000194bc3ca 621: 1< 0 - - > 2< 2543616 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131077:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000a4218c38 00000080:00200000:0.0:1769074780.131078:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 3. 00000080:00200000:0.0:1769074780.131080:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1134 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131094:0:63001:0:(rw26.c:912:ll_write_end()) pos 4644864, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131097:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000354173d2[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131099:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000062b1a756 17ffffc0001001 3:0 ffff9b03439e6288 1134 lru 00000080:00200000:0.0:1769074780.131103:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000e088986e 622: 1< 0 - - > 2< 2547712 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131106:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000354173d2 00000080:00200000:0.0:1769074780.131107:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 4. 00000080:00200000:0.0:1769074780.131109:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1135 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131122:0:63001:0:(rw26.c:912:ll_write_end()) pos 4648960, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131125:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000ad554bb2[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131128:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000044fff365 17ffffc0001001 3:0 ffff9b03439e60d8 1135 lru 00000080:00200000:0.0:1769074780.131133:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000007ed09cab 623: 1< 0 - - > 2< 2551808 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131135:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000ad554bb2 00000080:00200000:0.0:1769074780.131137:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 5. 00000080:00200000:0.0:1769074780.131138:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1136 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131162:0:63001:0:(rw26.c:912:ll_write_end()) pos 4653056, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131166:0:63001:0:(rw26.c:969:ll_write_end()) page@000000007c1d2e27[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131168:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000468eba69 17ffffc0001001 3:0 ffff9b0344c69288 1136 lru 00000080:00200000:0.0:1769074780.131173:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000aa249e9f 624: 1< 0 - - > 2< 2555904 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131175:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000007c1d2e27 00000080:00200000:0.0:1769074780.131176:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 6. 00000080:00200000:0.0:1769074780.131178:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1137 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131192:0:63001:0:(rw26.c:912:ll_write_end()) pos 4657152, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131196:0:63001:0:(rw26.c:969:ll_write_end()) page@000000009069f047[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131199:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000079529d2d 17ffffc0001001 3:0 ffff9b0344c690d8 1137 lru 00000080:00200000:0.0:1769074780.131204:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000b6c539a8 625: 1< 0 - - > 2< 2560000 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131206:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000009069f047 00000080:00200000:0.0:1769074780.131208:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 7. 00000080:00200000:0.0:1769074780.131209:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1138 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131230:0:63001:0:(rw26.c:912:ll_write_end()) pos 4661248, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131264:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000efead141[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131267:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000e4212c9f 17ffffc0001001 3:0 ffff9b0344c695e8 1138 lru 00000080:00200000:0.0:1769074780.131272:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000b53f0812 626: 1< 0 - - > 2< 2564096 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131275:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000efead141 00000080:00200000:0.0:1769074780.131276:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 8. 00000080:00200000:0.0:1769074780.131279:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1139 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131294:0:63001:0:(rw26.c:912:ll_write_end()) pos 4665344, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131298:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000087a6627f[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131300:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000443bf59f 17ffffc0001001 3:0 ffff9b0344c69ca8 1139 lru 00000080:00200000:0.0:1769074780.131305:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000003fd0e14f 627: 1< 0 - - > 2< 2568192 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131307:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000087a6627f 00000080:00200000:0.0:1769074780.131308:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 9. 00000080:00200000:0.0:1769074780.131310:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1140 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131325:0:63001:0:(rw26.c:912:ll_write_end()) pos 4669440, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131329:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000064bc5bb9[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131331:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000048927a90 17ffffc0001001 3:0 ffff9b0344c69000 1140 lru 00000080:00200000:0.0:1769074780.131337:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000d3b28468 628: 1< 0 - - > 2< 2572288 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131339:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000064bc5bb9 00000080:00200000:0.0:1769074780.131340:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 10. 00000080:00200000:0.0:1769074780.131342:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1141 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131356:0:63001:0:(rw26.c:912:ll_write_end()) pos 4673536, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131360:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000aac6492d[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131362:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000094be4769 17ffffc0001001 3:0 ffff9b0344c69948 1141 lru 00000080:00200000:0.0:1769074780.131367:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000003394780 629: 1< 0 - - > 2< 2576384 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131371:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000aac6492d 00000080:00200000:0.0:1769074780.131372:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 11. 00000080:00200000:0.0:1769074780.131374:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1142 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131388:0:63001:0:(rw26.c:912:ll_write_end()) pos 4677632, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131392:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000039ad3623[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131395:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000dbf50bfa 17ffffc0001001 3:0 ffff9b0344c69510 1142 lru 00000080:00200000:0.0:1769074780.131401:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000c7dee373 630: 1< 0 - - > 2< 2580480 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131403:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000039ad3623 00000080:00200000:0.0:1769074780.131404:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 12. 00000080:00200000:0.0:1769074780.131406:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1143 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131421:0:63001:0:(rw26.c:912:ll_write_end()) pos 4681728, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131426:0:63001:0:(rw26.c:969:ll_write_end()) page@000000000aed3153[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131428:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000951c116a 17ffffc0001001 3:0 ffff9b0344c69438 1143 lru 00000080:00200000:0.0:1769074780.131433:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000004522a03a 631: 1< 0 - - > 2< 2584576 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131435:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000000aed3153 00000080:00200000:0.0:1769074780.131436:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 13. 00000080:00200000:0.0:1769074780.131438:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1144 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131452:0:63001:0:(rw26.c:912:ll_write_end()) pos 4685824, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131455:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000062037067[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131459:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000ccf9bc2c 17ffffc0001001 3:0 ffff9b0344c69e58 1144 lru 00000080:00200000:0.0:1769074780.131464:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000003a01b26 632: 1< 0 - - > 2< 2588672 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131467:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000062037067 00000080:00200000:0.0:1769074780.131468:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 14. 00000080:00200000:0.0:1769074780.131470:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1145 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131483:0:63001:0:(rw26.c:912:ll_write_end()) pos 4689920, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131487:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000046ea17d3[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131490:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000624c227e 17ffffc0001001 3:0 ffff9b0344c69d80 1145 lru 00000080:00200000:0.0:1769074780.131494:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000f9ae421f 633: 1< 0 - - > 2< 2592768 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131496:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000046ea17d3 00000080:00200000:0.0:1769074780.131498:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 15. 00000080:00200000:0.0:1769074780.131499:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1146 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131513:0:63001:0:(rw26.c:912:ll_write_end()) pos 4694016, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131516:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000d6911407[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131519:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000004d02d0c4 17ffffc0001001 3:0 ffff9b0344c69af8 1146 lru 00000080:00200000:0.0:1769074780.131525:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000ac24c959 634: 1< 0 - - > 2< 2596864 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131527:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000d6911407 00000080:00200000:0.0:1769074780.131528:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 16. 00000080:00200000:0.0:1769074780.131530:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1147 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131544:0:63001:0:(rw26.c:912:ll_write_end()) pos 4698112, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131548:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000651ccd5e[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131550:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000002ea3387c 17ffffc0001001 3:0 ffff9b0344c69798 1147 lru 00000080:00200000:0.0:1769074780.131555:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000008f29dd46 635: 1< 0 - - > 2< 2600960 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131557:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000651ccd5e 00000080:00200000:0.0:1769074780.131558:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 17. 00000080:00200000:0.0:1769074780.131560:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1148 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131578:0:63001:0:(rw26.c:912:ll_write_end()) pos 4702208, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131582:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000026fee41[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131585:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000000e1b6364 17ffffc0001021 2:0 ffff9b0344c691b0 1148 lru 00000080:00200000:0.0:1769074780.131590:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000009799539a 636: 1< 0 - - > 2< 2605056 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131593:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000026fee41 00000080:00200000:0.0:1769074780.131594:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 18. 00000080:00200000:0.0:1769074780.131596:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1149 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131610:0:63001:0:(rw26.c:912:ll_write_end()) pos 4706304, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131614:0:63001:0:(rw26.c:969:ll_write_end()) page@000000003a1d788f[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131617:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000995e0575 17ffffc0001001 3:0 ffff9b0344c69a20 1149 lru 00000080:00200000:0.0:1769074780.131622:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000001cb3f8c6 637: 1< 0 - - > 2< 2609152 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131625:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000003a1d788f 00000080:00200000:0.0:1769074780.131626:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 19. 00000080:00200000:0.0:1769074780.131627:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1150 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131644:0:63001:0:(rw26.c:912:ll_write_end()) pos 4710400, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131648:0:63001:0:(rw26.c:969:ll_write_end()) page@000000006d4fde6a[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131651:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000ba23b334 17ffffc0001001 3:0 ffff9b0344c69360 1150 lru 00000080:00200000:0.0:1769074780.131656:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000a241625c 638: 1< 0 - - > 2< 2613248 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131659:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000006d4fde6a 00000080:00200000:0.0:1769074780.131660:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 20. 00000080:00200000:0.0:1769074780.131662:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1151 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131675:0:63001:0:(rw26.c:912:ll_write_end()) pos 4714496, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131679:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000049fe2c1c[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131681:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000006edcd12 17ffffc0001001 3:0 ffff9b0344c696c0 1151 lru 00000080:00200000:0.0:1769074780.131686:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000011c1520b 639: 1< 0 - - > 2< 2617344 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131688:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000049fe2c1c 00000080:00200000:0.0:1769074780.131689:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 21. 00000080:00200000:0.0:1769074780.131691:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1152 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131711:0:63001:0:(rw26.c:912:ll_write_end()) pos 4718592, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131715:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000b9e3b751[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131718:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000a20ee873 17ffffc0001001 3:0 ffff9b0344c69bd0 1152 lru 00000080:00200000:0.0:1769074780.131722:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000004306f15 640: 1< 0 - - > 2< 2621440 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131724:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000b9e3b751 00000080:00200000:0.0:1769074780.131725:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 22. 00000080:00200000:0.0:1769074780.131727:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1153 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131741:0:63001:0:(rw26.c:912:ll_write_end()) pos 4722688, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131745:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000a25f5267[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131748:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000a479a300 17ffffc0001001 3:0 ffff9b0344c69870 1153 lru 00000080:00200000:0.0:1769074780.131752:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000014dc1128 641: 1< 0 - - > 2< 2625536 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.131755:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000a25f5267 00000080:00200000:0.0:1769074780.131757:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 23. 00000080:00200000:0.0:1769074780.131758:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1154 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.131780:0:63001:0:(rw26.c:912:ll_write_end()) pos 4726784, len 4096, copied 4096 00000080:00200000:0.0:1769074780.131783:0:63001:0:(rw26.c:969:ll_write_end()) page@000000000138e922[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.131785:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000006631a340 17ffffc0001001 3:0 ffff9b0348df8000 1154 lru 00000080:00200000:0.0:1769074780.132958:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000db382aa6 642: 1< 0 - - > 2< 2629632 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.132964:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000000138e922 00000080:00200000:0.0:1769074780.132967:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 24. 00000080:00200000:0.0:1769074780.132980:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1155 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133009:0:63001:0:(rw26.c:912:ll_write_end()) pos 4730880, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133014:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000fc5c805b[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133017:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000091854ae9 17ffffc0001001 3:0 ffff9b0348df8948 1155 lru 00000080:00200000:0.0:1769074780.133021:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000644c4f35 643: 1< 0 - - > 2< 2633728 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133023:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000fc5c805b 00000080:00200000:0.0:1769074780.133025:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 25. 00000080:00200000:0.0:1769074780.133027:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1156 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133041:0:63001:0:(rw26.c:912:ll_write_end()) pos 4734976, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133044:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000650a8dba[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133046:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000043ffbdec 17ffffc0001001 3:0 ffff9b0348df8510 1156 lru 00000080:00200000:0.0:1769074780.133051:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000002c7a3814 644: 1< 0 - - > 2< 2637824 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133053:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000650a8dba 00000080:00200000:0.0:1769074780.133054:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 26. 00000080:00200000:0.0:1769074780.133056:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1157 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133071:0:63001:0:(rw26.c:912:ll_write_end()) pos 4739072, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133075:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000003871ed7[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133077:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000f8dcc693 17ffffc0001001 3:0 ffff9b0348df8438 1157 lru 00000080:00200000:0.0:1769074780.133082:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000ac38569b 645: 1< 0 - - > 2< 2641920 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133084:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000003871ed7 00000080:00200000:0.0:1769074780.133085:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 27. 00000080:00200000:0.0:1769074780.133087:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1158 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133102:0:63001:0:(rw26.c:912:ll_write_end()) pos 4743168, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133106:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000026d434c5[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133109:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000006666e0d 17ffffc0001001 3:0 ffff9b0348df8e58 1158 lru 00000080:00200000:0.0:1769074780.133114:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000004cf67903 646: 1< 0 - - > 2< 2646016 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133116:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000026d434c5 00000080:00200000:0.0:1769074780.133117:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 28. 00000080:00200000:0.0:1769074780.133119:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1159 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133133:0:63001:0:(rw26.c:912:ll_write_end()) pos 4747264, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133137:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000679d5769[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133140:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000008c1e6789 17ffffc0001001 3:0 ffff9b0348df8d80 1159 lru 00000080:00200000:0.0:1769074780.133144:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000feaa8da1 647: 1< 0 - - > 2< 2650112 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133146:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000679d5769 00000080:00200000:0.0:1769074780.133147:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 29. 00000080:00200000:0.0:1769074780.133150:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1160 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133165:0:63001:0:(rw26.c:912:ll_write_end()) pos 4751360, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133169:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000c3d0e205[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133172:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000006973e90b 17ffffc0001001 3:0 ffff9b0348df8af8 1160 lru 00000080:00200000:0.0:1769074780.133177:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000a3b66b70 648: 1< 0 - - > 2< 2654208 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133179:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000c3d0e205 00000080:00200000:0.0:1769074780.133180:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 30. 00000080:00200000:0.0:1769074780.133182:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1161 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133197:0:63001:0:(rw26.c:912:ll_write_end()) pos 4755456, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133201:0:63001:0:(rw26.c:969:ll_write_end()) page@000000004afb39bc[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133203:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000f7991354 17ffffc0001001 3:0 ffff9b0348df8798 1161 lru 00000080:00200000:0.0:1769074780.133208:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000008e043836 649: 1< 0 - - > 2< 2658304 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133209:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000004afb39bc 00000080:00200000:0.0:1769074780.133211:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 31. 00000080:00200000:0.0:1769074780.133213:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1162 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133229:0:63001:0:(rw26.c:912:ll_write_end()) pos 4759552, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133248:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000040366e4c[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133252:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000008a0af287 17ffffc0001001 3:0 ffff9b0348df81b0 1162 lru 00000080:00200000:0.0:1769074780.133257:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000ceca8479 650: 1< 0 - - > 2< 2662400 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133260:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000040366e4c 00000080:00200000:0.0:1769074780.133261:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 32. 00000080:00200000:0.0:1769074780.133263:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1163 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133284:0:63001:0:(rw26.c:912:ll_write_end()) pos 4763648, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133287:0:63001:0:(rw26.c:969:ll_write_end()) page@000000006a8853b0[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133290:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000033bdfa47 17ffffc0001021 2:0 ffff9b0348df8a20 1163 lru 00000080:00200000:0.0:1769074780.133295:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000d4dca17b 651: 1< 0 - - > 2< 2666496 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133298:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000006a8853b0 00000080:00200000:0.0:1769074780.133299:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 33. 00000080:00200000:0.0:1769074780.133301:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1164 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133317:0:63001:0:(rw26.c:912:ll_write_end()) pos 4767744, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133320:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000082109b89[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133323:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000287bf6d5 17ffffc0001001 3:0 ffff9b0348df8360 1164 lru 00000080:00200000:0.0:1769074780.133328:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000007cec54bb 652: 1< 0 - - > 2< 2670592 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133330:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000082109b89 00000080:00200000:0.0:1769074780.133331:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 34. 00000080:00200000:0.0:1769074780.133333:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1165 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133347:0:63001:0:(rw26.c:912:ll_write_end()) pos 4771840, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133351:0:63001:0:(rw26.c:969:ll_write_end()) page@000000008f85d426[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133353:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000fc31d360 17ffffc0001001 3:0 ffff9b0348df86c0 1165 lru 00000080:00200000:0.0:1769074780.133358:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000006bc6292 653: 1< 0 - - > 2< 2674688 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133359:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000008f85d426 00000080:00200000:0.0:1769074780.133361:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 35. 00000080:00200000:0.0:1769074780.133362:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1166 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133375:0:63001:0:(rw26.c:912:ll_write_end()) pos 4775936, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133379:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000042b21020[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133382:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000e7598301 17ffffc0001001 3:0 ffff9b0348df8bd0 1166 lru 00000080:00200000:0.0:1769074780.133387:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000499465fe 654: 1< 0 - - > 2< 2678784 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133389:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000042b21020 00000080:00200000:0.0:1769074780.133390:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 36. 00000080:00200000:0.0:1769074780.133392:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1167 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133405:0:63001:0:(rw26.c:912:ll_write_end()) pos 4780032, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133409:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000ff6cf1ae[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133412:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000001709922f 17ffffc0001001 3:0 ffff9b0348df8870 1167 lru 00000080:00200000:0.0:1769074780.133416:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000ecd31f45 655: 1< 0 - - > 2< 2682880 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133419:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000ff6cf1ae 00000080:00200000:0.0:1769074780.133420:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 37. 00000080:00200000:0.0:1769074780.133421:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1168 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133436:0:63001:0:(rw26.c:912:ll_write_end()) pos 4784128, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133439:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000263b6f6b[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133442:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000008ab04c17 17ffffc0001001 3:0 ffff9b0348df8288 1168 lru 00000080:00200000:0.0:1769074780.133447:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000042ef1518 656: 1< 0 - - > 2< 2686976 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133449:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000263b6f6b 00000080:00200000:0.0:1769074780.133450:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 38. 00000080:00200000:0.0:1769074780.133452:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1169 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133464:0:63001:0:(rw26.c:912:ll_write_end()) pos 4788224, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133468:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000b22b14fd[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133471:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000097f20037 17ffffc0001001 3:0 ffff9b0348df80d8 1169 lru 00000080:00200000:0.0:1769074780.133476:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000092827d12 657: 1< 0 - - > 2< 2691072 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133479:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000b22b14fd 00000080:00200000:0.0:1769074780.133479:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 39. 00000080:00200000:0.0:1769074780.133481:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1170 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133497:0:63001:0:(rw26.c:912:ll_write_end()) pos 4792320, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133501:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000052cb51dc[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133503:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000025d33ca6 17ffffc0001001 3:0 ffff9b0348df85e8 1170 lru 00000080:00200000:0.0:1769074780.133508:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000e4cbde23 658: 1< 0 - - > 2< 2695168 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133510:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000052cb51dc 00000080:00200000:0.0:1769074780.133511:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 40. 00000080:00200000:0.0:1769074780.133513:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1171 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133528:0:63001:0:(rw26.c:912:ll_write_end()) pos 4796416, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133532:0:63001:0:(rw26.c:969:ll_write_end()) page@0000000052ab6586[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133535:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000000ad80720 17ffffc0001001 3:0 ffff9b0348df8ca8 1171 lru 00000080:00200000:0.0:1769074780.133540:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000018403e4a 659: 1< 0 - - > 2< 2699264 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133542:0:63001:0:(rw26.c:969:ll_write_end()) end page@0000000052ab6586 00000080:00200000:0.0:1769074780.133543:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 41. 00000080:00200000:0.0:1769074780.133545:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1172 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133566:0:63001:0:(rw26.c:912:ll_write_end()) pos 4800512, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133571:0:63001:0:(rw26.c:969:ll_write_end()) page@000000005f1b5743[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133573:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000a2bae773 17ffffc0001001 3:0 ffff9b03653736c0 1172 lru 00000080:00200000:0.0:1769074780.133578:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000ca0d757b 660: 1< 0 - - > 2< 2703360 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133580:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000005f1b5743 00000080:00200000:0.0:1769074780.133582:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 42. 00000080:00200000:0.0:1769074780.133583:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1173 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133598:0:63001:0:(rw26.c:912:ll_write_end()) pos 4804608, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133602:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000bd2e1bcf[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133604:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000001503a627 17ffffc0001001 3:0 ffff9b0365373bd0 1173 lru 00000080:00200000:0.0:1769074780.133609:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000002f705fa3 661: 1< 0 - - > 2< 2707456 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133611:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000bd2e1bcf 00000080:00200000:0.0:1769074780.133612:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 43. 00000080:00200000:0.0:1769074780.133615:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1174 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133628:0:63001:0:(rw26.c:912:ll_write_end()) pos 4808704, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133632:0:63001:0:(rw26.c:969:ll_write_end()) page@000000007e161080[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133634:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000001766e435 17ffffc0001001 3:0 ffff9b0365373870 1174 lru 00000080:00200000:0.0:1769074780.133639:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000f7c3f0c1 662: 1< 0 - - > 2< 2711552 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133642:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000007e161080 00000080:00200000:0.0:1769074780.133643:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 44. 00000080:00200000:0.0:1769074780.133645:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1175 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133658:0:63001:0:(rw26.c:912:ll_write_end()) pos 4812800, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133662:0:63001:0:(rw26.c:969:ll_write_end()) page@000000009d694eb8[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133664:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000d3c597d3 17ffffc0001001 3:0 ffff9b0365373288 1175 lru 00000080:00200000:0.0:1769074780.133669:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000f91fedfe 663: 1< 0 - - > 2< 2715648 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133671:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000009d694eb8 00000080:00200000:0.0:1769074780.133672:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 45. 00000080:00200000:0.0:1769074780.133674:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1176 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133687:0:63001:0:(rw26.c:912:ll_write_end()) pos 4816896, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133691:0:63001:0:(rw26.c:969:ll_write_end()) page@000000007042895b[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133693:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000009ad558ac 17ffffc0001001 3:0 ffff9b03653730d8 1176 lru 00000080:00200000:0.0:1769074780.133698:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000019cf2507 664: 1< 0 - - > 2< 2719744 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133700:0:63001:0:(rw26.c:969:ll_write_end()) end page@000000007042895b 00000080:00200000:0.0:1769074780.133701:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 46. 00000080:00200000:0.0:1769074780.133702:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1177 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133713:0:63001:0:(rw26.c:912:ll_write_end()) pos 4820992, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133716:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000509a3efe[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133719:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @00000000ee7d6a1a 17ffffc0001001 3:0 ffff9b03653735e8 1177 lru 00000080:00200000:0.0:1769074780.133723:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@0000000065cbb344 665: 1< 0 - - > 2< 2723840 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133725:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000509a3efe 00000080:00200000:0.0:1769074780.133726:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 47. 00000080:00200000:0.0:1769074780.133727:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1178 of 0 to 4096 bytes 00000080:00200000:0.0:1769074780.133745:0:63001:0:(rw26.c:912:ll_write_end()) pos 4825088, len 4096, copied 4096 00000080:00200000:0.0:1769074780.133748:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000b78ad33c[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.133751:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @000000007a373a3a 17ffffc0001021 2:0 ffff9b0365373ca8 1178 lru 00000080:00200000:0.0:1769074780.133756:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@000000005247858d 666: 1< 0 - - > 2< 2727936 0 0 0x0 0x0 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 2 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.133758:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000b78ad33c 00000080:00200000:0.0:1769074780.133760:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 48. 00000080:00200000:0.0:1769074780.133761:0:63001:0:(rw26.c:729:ll_write_begin()) Writing 1179 of 0 to 3776 bytes 00000008:00010000:0.0:1769074780.135062:0:63001:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff9b03515b9b40 00000008:00010000:0.0:1769074780.135112:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9b40 00010000:00010000:0.0:1769074780.135140:0:28211:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2732032 2736127) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074780.135182:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a38e00 - object 0x2c0000401:703 offset 2732032<>2736128 00000008:00010000:0.0:1769074780.142599:0:63006:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff9b03515b9540 00000008:00010000:0.0:1769074780.142690:0:63006:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff9b03515b9d80 00010000:00010000:0.0:1769074780.142704:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.142720:0:63006:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.142730:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.142739:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.142756:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.142763:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.142773:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.142780:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.142811:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9d80 00010000:00010000:0.0:1769074780.142837:0:28211:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 1052671) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074780.142900:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a38380 - object 0x2c0000401:702 offset 1048576<>2097152 00000100:00100000:0.0:1769074780.142939:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074780.142952:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:543044d2-7a16-4e58-9b2c-28bb782e5779:2407:1855007616922368:192.168.204.160@tcp:3:cat.0 00000080:00200000:0.0:1769074780.205991:0:63001:0:(rw26.c:912:ll_write_end()) pos 4829184, len 3776, copied 3776 00000080:00200000:0.0:1769074780.206015:0:63001:0:(rw26.c:969:ll_write_end()) page@00000000157bafac[3 000000000669316c 2 1 000000005445278f] 00000080:00200000:0.0:1769074780.206019:0:63001:0:(rw26.c:969:ll_write_end()) vmpage @0000000037469f5f 17ffffc0001009 3:0 ffff9b0365373000 1179 lru 00000080:00200000:0.0:1769074780.206030:0:63001:0:(rw26.c:969:ll_write_end()) osc-page@00000000776c15c5 667: 1< 1 - - > 2< 2732032 0 4096 0x0 0x40009 | 00000000e7925d05 00000000957f7b33 > 3< 0 0 > 4< 1 0 8 484638720 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074780.206032:0:63001:0:(rw26.c:969:ll_write_end()) end page@00000000157bafac 00000080:00200000:0.0:1769074780.206033:0:63001:0:(rw26.c:969:ll_write_end()) queued page: 49. 00000080:00200000:0.0:1769074780.206043:0:63001:0:(vvp_io.c:1193:vvp_io_write_commit()) commit async pages: 49, from 1024, to 3776 prio 0 00000008:00010000:0.0:1769074780.206097:0:63001:0:(osc_cache.c:814:osc_extent_find()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206130:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206143:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206153:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206162:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206171:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206179:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206188:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206195:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206202:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206210:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206221:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206230:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206282:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206291:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.206313:0:63001:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce6578, count 15, dirtied 15 00000008:00010000:0.0:1769074780.206327:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206337:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206347:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206355:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206363:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206371:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206379:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206387:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206395:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206402:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206415:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206423:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206431:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206439:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206448:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.206460:0:63001:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce6578, count 15, dirtied 15 00000008:00010000:0.0:1769074780.206465:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206472:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206479:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206488:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206496:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206503:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206510:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206518:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206525:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206532:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206538:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206546:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206554:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206566:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.206573:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.206584:0:63001:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce6578, count 15, dirtied 15 00000008:00010000:0.0:1769074780.206589:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.210039:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.210087:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.210095:0:63001:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.210115:0:63001:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce6578, count 4, dirtied 4 00000080:00200000:0.0:1769074780.210124:0:63001:0:(vvp_io.c:1216:vvp_io_write_commit()) Committed 49 pages 199360 bytes, tot: 199360 00000080:00200000:0.0:1769074780.210130:0:63001:0:(vvp_io.c:1390:vvp_io_write_start()) f200b.sanity-flr: write bytes 0, result: 199360 00000020:00010000:0.0:1769074780.210160:0:63001:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0348dbe3f8 (ffff9b0342ca8908/0) at cl_lock_release():245 00000020:00010000:0.0:1769074780.210163:0:63001:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348dbe3f8 (ffff9b0342ca8908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.210168:0:63001:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359274c30 (ffff9b03515ed908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074780.210174:0:63001:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x810420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.210183:0:63001:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x810420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.210205:0:63001:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000020:00010000:0.0:1769074780.210208:0:63001:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348dbe3f8 (ffff9b0342ca8908/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.210211:0:63001:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359274c30 (ffff9b0342ca8908/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074780.210219:0:63001:0:(vvp_io.c:518:vvp_io_advance()) advancing 199360 bytes 00000080:00200000:0.0:1769074780.210225:0:63001:0:(file.c:2109:ll_file_io_generic()) Range unlock [1131, 1179] 00000080:00200000:0.0:1769074780.210247:0:63001:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 30 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074780.210252:0:63001:0:(file.c:2142:ll_file_io_generic()) f200b.sanity-flr: 2 io complete with rc: 0, result: 199360, restart: 0 00000080:00200000:0.0:1769074780.210264:0:63001:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 17576 00000080:00200000:0.0:1769074780.210270:0:63001:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 199360 00000080:00200008:0.0:1769074780.210297:0:63001:0:(file.c:424:ll_file_release()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), flags 100001 00000002:00100000:0.0:1769074780.210370:0:63001:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0365a38a80 x1855007616917504/t8589943996(8589943996) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/2240 e 0 to 0 dl 1769074796 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074780.210382:0:63001:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074780.210387:0:63001:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3a680 x1855007616922880/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074780.210409:0:63001:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a680 pname:cluuid:pid:xid:nid:opc:job multiop:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:63001:1855007616922880:192.168.204.160@tcp:35:multiop.0 00000100:00100000:0.0:1769074780.210486:0:63001:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8bd80 going to sleep for 11 seconds 00000008:00010000:0.0:1769074780.210518:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.210530:0:28211:0:(osc_cache.c:1103:osc_extent_make_ready()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.210616:0:28211:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2535424 2539519) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074780.210758:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a38e00 - object 0x2c0000401:703 offset 2535424<>2736128 00000020:00010000:0.0:1769074780.230981:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074780.230995:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.231001:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6b030 (ffff9b0353152a28/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074780.231012:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.231028:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074780.231044:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.231047:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6b030 (ffff9b03531527e8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074780.231059:0:63006:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:0.0:1769074780.231068:0:63006:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 1048576 chunk: [1048576, 2097152) 4194304, 1048576 00020000:00200000:0.0:1769074780.231071:0:63006:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:0.0:1769074780.231084:0:63006:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [256, 511] 00000008:00010000:0.0:1769074780.231099:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074780.231103:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df9b0, osc lock ffff9b0345a6db50, flags 0x400000000000000 00010000:00010000:0.0:1769074780.231120:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.231132:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.231150:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.231157:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.231164:0:63006:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.231172:0:63006:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [1048576, 2097152) 00000080:00200000:0.0:1769074780.231184:0:63006:0:(vvp_io.c:886:vvp_io_read_start()) count:1048576 iocb pos:1048576 00000008:00010000:0.0:1769074780.245678:0:2406:0:(osc_cache.c:861:osc_extent_finish()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800429400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.245896:0:2406:0:(osc_cache.c:483:__osc_extent_remove()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c1b7 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2535424->2736127) gid 0 flags: 0x800429400020000 nid: local remote: 0x7fe2323adfd54399 expref: -99 pid: 63001 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074780.246137:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616923008:192.168.204.160@tcp:4:cat.0 00000020:00010000:0.0:1769074780.247725:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074780.247733:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.247738:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df9b0 (ffff9b0353152e18/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074780.247747:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.247760:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074780.249402:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.249408:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df9b0 (ffff9b03531527e8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074780.249421:0:63006:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:0.0:1769074780.249597:0:63006:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 2 chunk: [2097152, 3145728) 4194304, 1048576 00020000:00200000:0.0:1769074780.249604:0:63006:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00000080:00200000:0.0:1769074780.249629:0:63006:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [512, 767] 00000008:00010000:0.0:1769074780.249652:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074780.249659:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df9b0, osc lock ffff9b0345a6db50, flags 0x400000000000000 00010000:00010000:0.0:1769074780.249683:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.249697:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.249711:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.249720:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.249727:0:63006:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.249737:0:63006:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [2097152, 3145728) 00000080:00200000:0.0:1769074780.249759:0:63006:0:(vvp_io.c:886:vvp_io_read_start()) count:1048576 iocb pos:2097152 00000100:00100000:0.0:1769074780.252986:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00010000:00010000:0.0:1769074780.278311:0:39340:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074780.278320:0:39340:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b036587b000/0xb7d6bf619879c1e8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2367488->2854911) gid 0 flags: 0x800420400020000 nid: local remote: 0x7fe2323adfd543ed expref: -99 pid: 63002 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.278328:0:39340:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b036587b000 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:0.0:1769074780.278335:0:39340:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b036587b000/0xb7d6bf619879c1e8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2367488->2854911) gid 0 flags: 0x800428400020000 nid: local remote: 0x7fe2323adfd543ed expref: -99 pid: 63002 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.309995:0:39340:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074780.310346:0:39340:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1769074780.310405:0:39340:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b036587b000/0xb7d6bf619879c1e8 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2367488->2854911) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd543ed expref: -99 pid: 63002 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.310415:0:39340:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074780.310441:0:39340:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b036587b000/0xb7d6bf619879c1e8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2367488->2854911) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd543ed expref: -99 pid: 63002 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.310450:0:39340:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b036587b000/0xb7d6bf619879c1e8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2367488->2854911) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd543ed expref: -99 pid: 63002 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.310466:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074780.310468:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074780.310470:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074780.310488:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:543044d2-7a16-4e58-9b2c-28bb782e5779:2406:1855007616924672:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074780.314194:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074780.316572:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074780.316635:0:55997:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074780.316653:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625179136 00000100:00100000:0.0:1769074780.316663:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625179136 00000100:00100000:0.0:1769074780.316720:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8861 00000100:00100000:0.0:1769074780.316730:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0343a29850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:36300:x1855007625179136:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074780.316745:0:55997:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 5/1,0 mode: --/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [2097152->3145727] (req 2097152->3145727) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.316756:0:55997:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 5/1,0 mode: --/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [2097152->3145727] (req 2097152->3145727) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.316764:0:55997:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 5/1,0 mode: --/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x10000000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.316776:0:55997:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x10080000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.316936:0:55997:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 6/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x10080000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.316949:0:55997:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 6/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x80000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.316980:0:55997:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b036587bc00) 00000100:00100000:0.0:1769074780.317035:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0343a29850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:36300:x1855007625179136:12345-192.168.204.160@tcp:105: Request processed in 307us (475us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074780.317043:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8861 00010000:00010000:0.0:1769074780.332014:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 6/3,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.332044:0:63007:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (3145728 3149823) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 6/3,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.337745:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 4/2,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.337772:0:63007:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (3145728 3149823) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 4/2,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.346856:0:63007:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff9b03515b9e40 00000008:00010000:0.0:1769074780.348059:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9e40 00010000:00010000:0.0:1769074780.348168:0:28211:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 8/3,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074780.348432:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a38000 - object 0x2c0000401:703 offset 2097152<>4194304 00000100:00100000:0.0:1769074780.348491:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616925056:192.168.204.160@tcp:3:cat.0 00000008:00010000:0.0:1769074780.348658:0:63007:0:(osc_cache.c:2941:osc_queue_sync_pages()) ### no dlm lock: extent: ffff9b03515b9000 00010000:00010000:0.0:1769074780.348681:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 8/3,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.348690:0:63007:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 7/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.348702:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.348714:0:63007:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.348722:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.348728:0:63007:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.348735:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.348744:0:63007:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074780.348770:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9000 00010000:00010000:0.0:1769074780.348811:0:28211:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 2101247) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074780.348911:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a39500 - object 0x280000401:700 offset 2097152<>4194304 00000100:00100000:0.0:1769074780.348937:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616925184:192.168.204.160@tcp:3:cat.0 00000008:00010000:0.0:1769074780.417699:0:2407:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b03515b9e40 00000020:00200000:0.0:1769074780.417742:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1769074780.417838:0:2407:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b03515b9e40 00000100:00100000:0.0:1769074780.417879:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616925056:192.168.204.160@tcp:3:cat.0 00000020:00010000:0.0:1769074780.432163:0:63007:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0352b7c308 (ffff9b0350057908/0) at cl_lock_release():245 00000020:00010000:0.0:1769074780.432175:0:63007:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352b7c308 (ffff9b0350057908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.432180:0:63007:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360af6fb0 (ffff9b0353152008/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074780.432193:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.432209:0:63007:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074780.432225:0:63007:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352b7c308 (ffff9b0350057908/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.432228:0:63007:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360af6fb0 (ffff9b0350057908/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074780.432278:0:63007:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:0.0:1769074780.432290:0:63007:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 6 chunk: [6291456, 7340032) 8388608, 1048576 00020000:00200000:0.0:1769074780.432295:0:63007:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00000080:00200000:0.0:1769074780.432312:0:63007:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [1536, 1791] 00000008:00010000:0.0:1769074780.432333:0:63007:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074780.432338:0:63007:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df930, osc lock ffff9b0345a6cbe0, flags 0x400000000000000 00010000:00010000:0.0:1769074780.432362:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.432378:0:63007:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.432391:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.432399:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.432411:0:63007:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.432420:0:63007:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [6291456, 7340032) 00000080:00200000:0.0:1769074780.432438:0:63007:0:(vvp_io.c:886:vvp_io_read_start()) count:1048576 iocb pos:6291456 00000020:00010000:0.0:1769074780.433529:0:63007:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0352b7c308 (ffff9b0350057908/0) at cl_lock_release():245 00000020:00010000:0.0:1769074780.433540:0:63007:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352b7c308 (ffff9b0350057908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.433543:0:63007:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df930 (ffff9b0342ca6518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074780.433551:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.433564:0:63007:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b036587bc00/0xb7d6bf619879c1ef lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543f4 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074780.433579:0:63007:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352b7c308 (ffff9b0350057908/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.433582:0:63007:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df930 (ffff9b0350057908/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074780.433594:0:63007:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:0.0:1769074780.433601:0:63007:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 7 chunk: [7340032, 8388608) 8388608, 1048576 00020000:00200000:0.0:1769074780.433605:0:63007:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00000080:00200000:0.0:1769074780.433618:0:63007:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [1792, 2047] 00000008:00010000:0.0:1769074780.433632:0:63007:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074780.433636:0:63007:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df930, osc lock ffff9b0345a6cbe0, flags 0x400000000000000 00010000:00010000:0.0:1769074780.433652:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.433666:0:63007:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.433680:0:63007:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.433687:0:63007:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.433694:0:63007:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.433702:0:63007:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [7340032, 8388608) 00000080:00200000:0.0:1769074780.433716:0:63007:0:(vvp_io.c:886:vvp_io_read_start()) count:1048576 iocb pos:7340032 00000100:00100000:0.0:1769074780.447955:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616925312:192.168.204.160@tcp:3:cat.0 00000008:00010000:0.0:1769074780.478298:0:2407:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b0347fb6cc0 00000020:00200000:0.0:1769074780.478872:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1769074780.478928:0:2407:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b0347fb6cc0 00000100:00100000:0.0:1769074780.478974:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616925312:192.168.204.160@tcp:3:cat.0 00000020:00010000:0.0:1769074780.484989:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074780.484994:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.484998:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6ba30 (ffff9b0342ca6c68/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074780.485005:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485015:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074780.485033:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.485035:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6ba30 (ffff9b03531527e8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074780.485045:0:63006:0:(vvp_io.c:518:vvp_io_advance()) advancing 611424 bytes 00000080:00200000:0.0:1769074780.485055:0:63006:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 30 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074780.485060:0:63006:0:(file.c:2142:ll_file_io_generic()) f200b.sanity-flr: 1 io complete with rc: 0, result: 611424, restart: 0 00000080:00200000:0.0:1769074780.485064:0:63006:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 611424 00000080:00200008:0.0:1769074780.485087:0:63006:0:(file.c:2474:do_file_read_iter()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0], ppos: 9000032, count: 4194304 00000080:00200000:0.0:1769074780.485094:0:63006:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 30 restore needed 0 00020000:00200000:0.0:1769074780.485101:0:63006:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074780.485104:0:63006:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:0.0:1769074780.485111:0:63006:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 30 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074780.485116:0:63006:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 30 restore needed 0 00020000:00200000:0.0:1769074780.485120:0:63006:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074780.485123:0:63006:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1769074780.485124:0:63006:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fb:0x0] 00000008:00010000:0.0:1769074780.485141:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.485144:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0345740430, osc lock ffff9b0345a6dc80, flags 0x400000400001002 00000008:00010000:0.0:1769074780.485153:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.485155:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0345740470, osc lock ffff9b0345a6d7c0, flags 0x400000400001002 00000008:00010000:0.0:1769074780.485160:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.485161:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03457404b0, osc lock ffff9b0345a6da20, flags 0x400000400001002 00000008:00010000:0.0:1769074780.485167:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.485169:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03457404f0, osc lock ffff9b0345a6c390, flags 0x400000400001002 00000008:00010000:0.0:1769074780.485175:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.485177:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0345740530, osc lock ffff9b0345a6cab0, flags 0x400000400001002 00000008:00010000:0.0:1769074780.485183:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.485185:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0345740570, osc lock ffff9b0345a6c980, flags 0x400000400001002 00010000:00010000:0.0:1769074780.485200:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485211:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.485220:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.485223:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485229:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485263:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485273:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.485281:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.485283:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485291:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485303:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485870:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.485878:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.485881:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485888:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485904:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485911:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.485917:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.485920:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485926:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485936:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485946:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 8388607) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.485952:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.485954:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485960:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485982:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341219e00/0xb7d6bf619879c1e1 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543df expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.485989:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 8388607) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341219e00/0xb7d6bf619879c1e1 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543df expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.485997:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.485999:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341219e00/0xb7d6bf619879c1e1 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543df expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.486007:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341219e00/0xb7d6bf619879c1e1 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543df expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.486013:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074780.486024:0:63006:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 17576 00000020:00010000:0.0:1769074780.486028:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0353501da8 (ffff9b0342ca6c68/0) at cl_lock_release():245 00000020:00010000:0.0:1769074780.486031:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0353501da8 (ffff9b0342ca6c68/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.486033:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0345740430 (ffff9b0342ca8908/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.486036:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0345740470 (ffff9b03515ed908/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.486039:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03457404b0 (ffff9b03591bed88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.486041:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03457404f0 (ffff9b036593a5a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.486043:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0345740530 (ffff9b03515edbd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.486045:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0345740570 (ffff9b0342ca8e18/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.486048:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0353501da8 (ffff9b0342ca6c68/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.486050:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0345740430 (ffff9b0342ca6c68/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.486054:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0345740470 (ffff9b0342ca6c68/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.486056:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03457404b0 (ffff9b0342ca6c68/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.486058:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03457404f0 (ffff9b0342ca6c68/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.486060:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0345740530 (ffff9b0342ca6c68/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.486062:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0345740570 (ffff9b0342ca6c68/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074780.486076:0:63006:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 30 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074780.486081:0:63006:0:(file.c:2367:file_read_confine_iter()) f200b.sanity-flr: read [9000032, 13194336] over eof, kms 9000032, file_size 9000032. 00000080:00200000:0.0:1769074780.486095:0:63006:0:(file.c:1994:ll_file_io_generic()) f200b.sanity-flr: read ppos: 9000032, bytes: 4194304 00000080:00200000:0.0:1769074780.486099:0:63006:0:(file.c:1759:ll_io_set_mirror()) f200b.sanity-flr: desiginated mirror: 0 00000020:00200000:0.0:1769074780.486106:0:63006:0:(cl_io.c:214:cl_io_rw_init()) header@000000009fb57c89[0x4, 2199, [0x200000405:0x1fb:0x0] hash] 00000020:00200000:0.0:1769074780.486108:0:63006:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [9000032, 13194336) 0 0 00000080:00200000:0.0:1769074780.486111:0:63006:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 30 restore needed 0 00020000:00200000:0.0:1769074780.486120:0:63006:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074780.486122:0:63006:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00020000:00200000:0.0:1769074780.486128:0:63006:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 8 chunk: [9000032, 9437184) 13194336, 437152 00020000:00200000:0.0:1769074780.486132:0:63006:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00000080:00200000:0.0:1769074780.486146:0:63006:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [2197, 2303] 00000008:00010000:0.0:1769074780.486160:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074780.486164:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df0b0, osc lock ffff9b0345a6c980, flags 0x400000000000000 00010000:00010000:0.0:1769074780.486172:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.486181:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (4804608 5242879) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.486189:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.486196:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.486204:0:63006:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074780.486211:0:63006:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [9000032, 9437184) 00000001:00010000:0.0:1769074780.486222:0:63006:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fb:0x0] 00000008:00010000:0.0:1769074780.486230:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.486258:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360990e30, osc lock ffff9b0345a6cab0, flags 0x400000400001002 00000008:00010000:0.0:1769074780.486264:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.486265:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360990e70, osc lock ffff9b0345a6c390, flags 0x400000400001002 00000008:00010000:0.0:1769074780.486271:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.486272:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360990eb0, osc lock ffff9b0345a6da20, flags 0x400000400001002 00000008:00010000:0.0:1769074780.486276:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.486278:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360990ef0, osc lock ffff9b0345a6d7c0, flags 0x400000400001002 00000008:00010000:0.0:1769074780.486281:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.486282:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360990f30, osc lock ffff9b0345a6dc80, flags 0x400000400001002 00000008:00010000:0.0:1769074780.486286:0:63006:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074780.486288:0:63006:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360990f70, osc lock ffff9b0345a6cf70, flags 0x400000400001002 00010000:00010000:0.0:1769074780.486295:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.486308:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.487449:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.487453:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487461:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0350988e00/0xb7d6bf619879c1be lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543c3 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487483:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487493:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.487500:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.487502:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487510:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0350988800/0xb7d6bf619879c1c5 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543d1 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487522:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487536:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.487544:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.487546:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487553:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341218400/0xb7d6bf619879c1cc lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543ca expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487567:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487575:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.487582:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.487585:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487591:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0341219c00/0xb7d6bf619879c1d3 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543e6 expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487603:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 4/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487614:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 8388607) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 4/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.487620:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.487622:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 7/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487628:0:63006:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 6/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487749:0:63006:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341219e00/0xb7d6bf619879c1e1 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543df expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.487763:0:63006:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 8388607) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341219e00/0xb7d6bf619879c1e1 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543df expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.487771:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074780.487775:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341219e00/0xb7d6bf619879c1e1 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd543df expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.488109:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b03593e7000 lock: ffff9b0341219e00/0xb7d6bf619879c1e1 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd543df expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074780.488120:0:63006:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074780.488136:0:63006:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 17576 00000020:00010000:0.0:1769074780.488141:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314efe8 (ffff9b03531527e8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074780.488144:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314efe8 (ffff9b03531527e8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.488147:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360990e30 (ffff9b0342ca8e18/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.488151:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360990e70 (ffff9b03515edbd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.488153:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360990eb0 (ffff9b036593a5a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.488157:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360990ef0 (ffff9b03591bed88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.488159:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360990f30 (ffff9b0342ca6c68/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.488161:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360990f70 (ffff9b03515ed908/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.488164:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314efe8 (ffff9b03531527e8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.488166:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360990e30 (ffff9b03531527e8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.488171:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360990e70 (ffff9b03531527e8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.488173:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360990eb0 (ffff9b03531527e8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.488175:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360990ef0 (ffff9b03531527e8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.488178:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360990f30 (ffff9b03531527e8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.488180:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360990f70 (ffff9b03531527e8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074780.488192:0:63006:0:(vvp_io.c:886:vvp_io_read_start()) count:437152 iocb pos:9000032 00000020:00010000:0.0:1769074780.488209:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074780.488211:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074780.488214:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df0b0 (ffff9b0342ca6c68/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074780.488219:0:63006:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 4/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074780.488226:0:63006:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b03593e7000 lock: ffff9b0342cd1600/0xb7d6bf619879c1f6 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd543fb expref: -99 pid: 63006 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074780.488260:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03451ff128 (ffff9b03531527e8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074780.488264:0:63006:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df0b0 (ffff9b03531527e8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074780.488272:0:63006:0:(vvp_io.c:518:vvp_io_advance()) advancing 0 bytes 00000080:00200000:0.0:1769074780.488295:0:63006:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 30 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074780.488300:0:63006:0:(file.c:2142:ll_file_io_generic()) f200b.sanity-flr: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:0.0:1769074780.488303:0:63006:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:0.0:1769074780.493499:0:63006:0:(file.c:424:ll_file_release()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0](ffff9b03603963c8), flags 100000 00000002:00100000:0.0:1769074780.493570:0:63006:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0342cea680 x1855007616918272/t8589943998(8589943998) o101->lustre-MDT0000-mdc-ffff9b03593e7000@192.168.204.160@tcp:12/10 lens 2152/2240 e 0 to 0 dl 1769074796 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074780.493583:0:63006:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b03593e7000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074780.493589:0:63006:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a38000 x1855007616925952/t0(0) o35->lustre-MDT0000-mdc-ffff9b03593e7000@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074780.493608:0:63006:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job cat:543044d2-7a16-4e58-9b2c-28bb782e5779:63006:1855007616925952:192.168.204.160@tcp:35:cat.0 00000100:00100000:0.0:1769074780.493669:0:63006:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b300 going to sleep for 11 seconds 00000100:00100000:0.0:1769074780.499156:0:63006:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job cat:543044d2-7a16-4e58-9b2c-28bb782e5779:63006:1855007616925952:192.168.204.160@tcp:35:cat.0 00000002:00100000:0.0:1769074780.499201:0:63006:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0342cea680 x1855007616918272/t8589943998(8589943998) o101->lustre-MDT0000-mdc-ffff9b03593e7000@192.168.204.160@tcp:12/10 lens 2152/2240 e 0 to 0 dl 1769074796 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'cat.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1769074782.611533:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000080:0.0:1769074783.565483:0:36:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff9b03452f1800 for lustre-OST0001-osc-ffff9b03593e7000 00000020:00000080:0.0:1769074783.565491:0:36:0:(genops.c:904:class_export_put()) final put ffff9b03452f3800/543044d2-7a16-4e58-9b2c-28bb782e5779 00000020:00000080:0.0:1769074783.565504:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff9b03452f3800/543044d2-7a16-4e58-9b2c-28bb782e5779 for lustre-OST0001-osc-ffff9b03593e7000 00000020:01000000:0.0:1769074783.565515:0:36:0:(genops.c:443:class_free_dev()) finishing cleanup of obd lustre-OST0001-osc-ffff9b03593e7000 (543044d2-7a16-4e58-9b2c-28bb782e5779) 00000020:00000080:0.0:1769074783.573426:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff9b03452f7000/LOV_OSC_UUID for lustre-OST0000-osc-ffff9b03593e7000 00000020:00000080:0.0:1769074783.575524:0:36:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff9b03452f6800 for lustre-OST0000-osc-ffff9b03593e7000 00000020:00000080:0.0:1769074783.575533:0:36:0:(genops.c:904:class_export_put()) final put ffff9b03452f2800/543044d2-7a16-4e58-9b2c-28bb782e5779 00000020:00000080:0.0:1769074783.575537:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff9b03452f2800/543044d2-7a16-4e58-9b2c-28bb782e5779 for lustre-OST0000-osc-ffff9b03593e7000 00000020:01000000:0.0:1769074783.575543:0:36:0:(genops.c:443:class_free_dev()) finishing cleanup of obd lustre-OST0000-osc-ffff9b03593e7000 (543044d2-7a16-4e58-9b2c-28bb782e5779) 00000020:00000080:0.0:1769074783.575796:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff9b03452f2000/543044d2-7a16-4e58-9b2c-28bb782e5779 for lustre-clilov-ffff9b03593e7000 00000100:00100000:0.0:1769074783.590906:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:543044d2-7a16-4e58-9b2c-28bb782e5779:2405:1855007616927872:192.168.204.160@tcp:39:umount.0 00000020:00000080:0.0:1769074783.619634:0:63031:0:(genops.c:1485:class_disconnect()) disconnect: cookie 0xb7d6bf6198798e25 00000020:00000080:0.0:1769074783.619640:0:63031:0:(genops.c:904:class_export_put()) final put ffff9b0342f85000/543044d2-7a16-4e58-9b2c-28bb782e5779 00000020:01000000:0.0:1769074783.624053:0:63031:0:(obd_config.c:2361:class_manual_cleanup()) Manual cleanup of lustre-clilov-ffff9b03593e7000 (flags='') 00000020:00000080:0.0:1769074783.624070:0:63031:0:(obd_config.c:1389:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1769074783.624078:0:63031:0:(obd_config.c:886:class_cleanup()) lustre-clilov-ffff9b03593e7000: forcing exports to disconnect: 0/1 00000020:00080000:0.0:1769074783.624083:0:63031:0:(genops.c:1575:class_disconnect_exports()) OBD device 7 (ffff9b03512d8000) has no exports 00000020:00000080:0.0:1769074783.624138:0:63031:0:(obd_config.c:1389:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1769074783.624145:0:63031:0:(obd_config.c:814:class_detach()) detach on obd lustre-clilov-ffff9b03593e7000 (uuid 543044d2-7a16-4e58-9b2c-28bb782e5779) 00000020:00000080:0.0:1769074783.624150:0:63031:0:(genops.c:904:class_export_put()) final put ffff9b03452f4800/543044d2-7a16-4e58-9b2c-28bb782e5779 00000020:00000080:0.0:1769074783.624154:0:63031:0:(genops.c:863:class_export_destroy()) destroying export ffff9b03452f4800/543044d2-7a16-4e58-9b2c-28bb782e5779 for lustre-clilov-ffff9b03593e7000 00000020:01000000:0.0:1769074783.624168:0:63031:0:(genops.c:443:class_free_dev()) finishing cleanup of obd lustre-clilov-ffff9b03593e7000 (543044d2-7a16-4e58-9b2c-28bb782e5779) 00000020:01000000:0.0:1769074783.624177:0:63031:0:(obd_config.c:2361:class_manual_cleanup()) Manual cleanup of lustre-clilmv-ffff9b03593e7000 (flags='') 00000020:00000080:0.0:1769074783.624188:0:63031:0:(obd_config.c:1389:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1769074783.624192:0:63031:0:(obd_config.c:886:class_cleanup()) lustre-clilmv-ffff9b03593e7000: forcing exports to disconnect: 0/2 00000020:00080000:0.0:1769074783.624194:0:63031:0:(genops.c:1575:class_disconnect_exports()) OBD device 8 (ffff9b03512dca88) has no exports 00000020:00000080:0.0:1769074783.624284:0:63031:0:(obd_config.c:1389:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1769074783.624288:0:63031:0:(obd_config.c:814:class_detach()) detach on obd lustre-clilmv-ffff9b03593e7000 (uuid 543044d2-7a16-4e58-9b2c-28bb782e5779) 00000020:01000000:0.0:1769074783.624291:0:63031:0:(obd_config.c:2361:class_manual_cleanup()) Manual cleanup of lustre-MDT0000-mdc-ffff9b03593e7000 (flags='') 00000020:00000080:0.0:1769074783.624302:0:63031:0:(obd_config.c:1389:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1769074783.624305:0:63031:0:(obd_config.c:886:class_cleanup()) lustre-MDT0000-mdc-ffff9b03593e7000: forcing exports to disconnect: 0/2 00000020:00080000:0.0:1769074783.624308:0:63031:0:(genops.c:1575:class_disconnect_exports()) OBD device 9 (ffff9b0360b30000) has no exports 00000008:01000000:0.0:1769074783.624948:0:63031:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-MDT0000-mdc-ffff9b03593e7000: client import never connected 00000100:00080000:0.0:1769074783.624954:0:63031:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00100000:0.0:1769074783.624959:0:63031:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b03593e7000: committing for last_committed 8589943998 gen 3 00000100:00020000:0.0:1769074783.624964:0:63031:0:(obd_class.h:479:obd_check_dev()) Device 9 not setup 00000100:00100000:0.0:1769074783.624967:0:63031:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00020000:0.0:1769074783.624970:0:63031:0:(obd_class.h:479:obd_check_dev()) Device 9 not setup 00000020:00000080:0.0:1769074783.625499:0:63031:0:(obd_config.c:1389:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1769074783.625509:0:63031:0:(obd_config.c:814:class_detach()) detach on obd lustre-MDT0000-mdc-ffff9b03593e7000 (uuid 543044d2-7a16-4e58-9b2c-28bb782e5779) 00000020:01000000:0.0:1769074783.625513:0:63031:0:(obd_config.c:2361:class_manual_cleanup()) Manual cleanup of lustre-MDT0001-mdc-ffff9b03593e7000 (flags='') 00000020:00000080:0.0:1769074783.625528:0:63031:0:(obd_config.c:1389:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1769074783.625531:0:63031:0:(obd_config.c:886:class_cleanup()) lustre-MDT0001-mdc-ffff9b03593e7000: forcing exports to disconnect: 0/2 00000020:00080000:0.0:1769074783.625535:0:63031:0:(genops.c:1575:class_disconnect_exports()) OBD device 10 (ffff9b0360b318d8) has no exports 00000008:01000000:0.0:1769074783.630999:0:63031:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-MDT0001-mdc-ffff9b03593e7000: client import never connected 00000100:00080000:0.0:1769074783.631011:0:63031:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0001_UUID INVALID 00000100:00100000:0.0:1769074783.631016:0:63031:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9b03593e7000: committing for last_committed 0 gen 3 00000100:00020000:0.0:1769074783.631022:0:63031:0:(obd_class.h:479:obd_check_dev()) Device 10 not setup 00000100:00100000:0.0:1769074783.631026:0:63031:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00020000:0.0:1769074783.631028:0:63031:0:(obd_class.h:479:obd_check_dev()) Device 10 not setup 00010000:00010000:0.0:1769074783.639205:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074785.368117:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616928128:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.368216:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616928384:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.368262:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:2407:1855007616928640:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.368287:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:2407:1855007616928896:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.368318:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 5 async RPCs [0->1] 00000100:00100000:0.0:1769074785.368323:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007616928000:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.368346:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616928256:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.368364:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616928512:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.368398:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:2407:1855007616928768:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.368418:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:2407:1855007616929024:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.379640:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616928256:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.379659:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616928512:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.379671:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:2407:1855007616928768:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.379682:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:2407:1855007616929024:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.379692:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616928128:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.379702:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616928384:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074785.379713:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:2407:1855007616928896:192.168.204.160@tcp:400:kworker.0 00000080:00200008:0.0:1769074786.761026:0:63051:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0355ce6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074786.761046:0:63051:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0355ce6c08),name=/ 00000002:00010000:0.0:1769074786.761078:0:63051:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074786.761102:0:63051:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074786.761122:0:63051:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074786.761131:0:63051:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074786.761143:0:63051:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0355ce6c08) 00000080:00010000:0.0:1769074786.761148:0:63051:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0355ce6c08) for lock 0xb7d6bf6198798eb8 00000080:00010000:0.0:1769074786.761157:0:63051:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf6198798eb8 from it ffffb43642f77cb8 00010000:00010000:0.0:1769074786.761161:0:63051:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074786.761171:0:63051:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00000080:00200008:0.0:1769074786.761604:0:63051:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0355ce6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074786.761610:0:63051:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0355ce6c08),name=/ 00000002:00010000:0.0:1769074786.761619:0:63051:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074786.761627:0:63051:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074786.761637:0:63051:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074786.761642:0:63051:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074786.761647:0:63051:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0355ce6c08) 00000080:00010000:0.0:1769074786.761649:0:63051:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0355ce6c08) for lock 0xb7d6bf6198798eb8 00000080:00010000:0.0:1769074786.761654:0:63051:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf6198798eb8 from it ffffb43642f77cb8 00010000:00010000:0.0:1769074786.761656:0:63051:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074786.761663:0:63051:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00000100:00020000:0.0:1769074786.840719:0:63051:0:(obd_class.h:479:obd_check_dev()) Device 18 not setup 00000020:00000080:0.0:1769074786.841437:0:63051:0:(obd_config.c:1389:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1769074786.841448:0:63051:0:(obd_config.c:814:class_detach()) detach on obd lustre-OST0001-osc-ffff9b0348feb000 (uuid 0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b) 00000020:00000080:0.0:1769074786.841453:0:63051:0:(genops.c:904:class_export_put()) final put ffff9b0342f80000/0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b 00000020:00000080:0.0:1769074786.841457:0:63051:0:(genops.c:863:class_export_destroy()) destroying export ffff9b0342f80000/0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b for lustre-OST0001-osc-ffff9b0348feb000 00000020:01000000:0.0:1769074786.841462:0:63051:0:(genops.c:443:class_free_dev()) finishing cleanup of obd lustre-OST0001-osc-ffff9b0348feb000 (0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b) 00020000:01000000:0.0:1769074786.841656:0:63051:0:(lov_obd.c:611:__lov_del_obd()) Removing tgt lustre-OST0000_UUID : lustre-OST0000-osc-ffff9b0348feb000 00020000:01000000:0.0:1769074786.841659:0:63051:0:(lov_obd.c:248:lov_disconnect_obd()) lustre-clilov-ffff9b0348feb000: disconnecting target lustre-OST0000-osc-ffff9b0348feb000 00000100:00080000:0.0:1769074786.841665:0:63051:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b->lustre-OST0000_UUID 00010000:00010000:0.0:1769074786.841690:0:63051:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350464a00/0xb7d6bf619879c12b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd542b2 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.841748:0:63051:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350464a00/0xb7d6bf619879c12b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd542b2 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.841755:0:63051:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074786.841772:0:63051:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b180 going to sleep for 0 seconds 00000100:00100000:0.0:1769074786.841782:0:63051:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job umount:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:63051:1855007616930688:192.168.204.160@tcp:103:umount.0 00010000:00010000:0.0:1769074786.841790:0:63051:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1769074786.842043:0:63051:0:(ldlm_request.c:2263:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1769074786.842048:0:63051:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350464a00/0xb7d6bf619879c12b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2bf:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd542b2 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.842070:0:63051:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0340030200/0xb7d6bf6198799fac lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd50c48 expref: -99 pid: 62258 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.842089:0:63051:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0340030200/0xb7d6bf6198799fac lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd50c48 expref: -99 pid: 62258 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.842094:0:63051:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074786.842100:0:63051:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b180 going to sleep for 0 seconds 00000100:00100000:0.0:1769074786.842105:0:63051:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job umount:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:63051:1855007616930816:192.168.204.160@tcp:103:umount.0 00010000:00010000:0.0:1769074786.842111:0:63051:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1769074786.842114:0:63051:0:(ldlm_request.c:2263:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1769074786.842117:0:63051:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0340030200/0xb7d6bf6198799fac lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2c0:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd50c48 expref: -99 pid: 62258 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.842153:0:63051:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350464200/0xb7d6bf619879c108 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd5428f expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.842171:0:63051:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350464200/0xb7d6bf619879c108 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd5428f expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.842177:0:63051:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074786.842183:0:63051:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b180 going to sleep for 0 seconds 00000100:00100000:0.0:1769074786.842187:0:63051:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job umount:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:63051:1855007616930944:192.168.204.160@tcp:103:umount.0 00010000:00010000:0.0:1769074786.842191:0:63051:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1769074786.842195:0:63051:0:(ldlm_request.c:2263:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1769074786.842197:0:63051:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350464200/0xb7d6bf619879c108 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd5428f expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.842211:0:63051:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0344ccda00/0xb7d6bf619879c0d7 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd541cb expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.842225:0:63051:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0344ccda00/0xb7d6bf619879c0d7 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd541cb expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.842231:0:63051:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074786.846273:0:63051:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b180 going to sleep for 0 seconds 00000100:00100000:0.0:1769074786.846290:0:63051:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job umount:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:63051:1855007616931072:192.168.204.160@tcp:103:umount.0 00010000:00010000:0.0:1769074786.846304:0:63051:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1769074786.846316:0:63051:0:(ldlm_request.c:2263:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1769074786.846323:0:63051:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0344ccda00/0xb7d6bf619879c0d7 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd541cb expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846363:0:63051:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0344cccc00/0xb7d6bf619879c0de lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd541d2 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846411:0:63051:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0344cccc00/0xb7d6bf619879c0de lrc: 1/0,0 mode: --/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd541d2 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846419:0:63051:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074786.846425:0:63051:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b180 going to sleep for 0 seconds 00000100:00100000:0.0:1769074786.846430:0:63051:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job umount:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:63051:1855007616931200:192.168.204.160@tcp:103:umount.0 00010000:00010000:0.0:1769074786.846435:0:63051:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1769074786.846439:0:63051:0:(ldlm_request.c:2263:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1769074786.846442:0:63051:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0344cccc00/0xb7d6bf619879c0de lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd541d2 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846458:0:63051:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0344ccd800/0xb7d6bf619879c0f3 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd541e7 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846475:0:63051:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0344ccd800/0xb7d6bf619879c0f3 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd541e7 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846481:0:63051:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074786.846485:0:63051:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b180 going to sleep for 0 seconds 00000100:00100000:0.0:1769074786.846490:0:63051:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job umount:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:63051:1855007616931328:192.168.204.160@tcp:103:umount.0 00010000:00010000:0.0:1769074786.846495:0:63051:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1769074786.846498:0:63051:0:(ldlm_request.c:2263:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1769074786.846500:0:63051:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0344ccd800/0xb7d6bf619879c0f3 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd541e7 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846517:0:63051:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350465000/0xb7d6bf619879c10f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd54296 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846533:0:63051:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350465000/0xb7d6bf619879c10f lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd54296 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846539:0:63051:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074786.846543:0:63051:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b180 going to sleep for 0 seconds 00000100:00100000:0.0:1769074786.846547:0:63051:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job umount:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:63051:1855007616931456:192.168.204.160@tcp:103:umount.0 00010000:00010000:0.0:1769074786.846551:0:63051:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1769074786.846554:0:63051:0:(ldlm_request.c:2263:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1769074786.846556:0:63051:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350465000/0xb7d6bf619879c10f lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2bd:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd54296 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846573:0:63051:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350465600/0xb7d6bf619879c11d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd542a4 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846590:0:63051:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350465600/0xb7d6bf619879c11d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd542a4 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074786.846596:0:63051:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074786.846601:0:63051:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b180 going to sleep for 0 seconds 00000100:00100000:0.0:1769074786.846604:0:63051:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job umount:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:63051:1855007616931584:192.168.204.160@tcp:103:umount.0 00010000:00010000:0.0:1769074786.846609:0:63051:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1769074786.846612:0:63051:0:(ldlm_request.c:2263:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1769074786.846614:0:63051:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0348feb000 lock: ffff9b0350465600/0xb7d6bf619879c11d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2be:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd542a4 expref: -99 pid: 62993 timeout: 0 lvb_type: 1 00000100:00080000:0.0:1769074786.846696:0:63051:0:(import.c:65:import_set_state_nolock()) ffff9b0342f80800 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:00080000:0.0:1769074786.859123:0:63051:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00100000:0.0:1769074786.859128:0:63051:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9b0348feb000: committing for last_committed 21474841223 gen 2 00000100:00100000:0.0:1769074786.859133:0:63051:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:0.0:1769074786.859317:0:63051:0:(genops.c:1485:class_disconnect()) disconnect: cookie 0xb7d6bf6198798eaa 00000020:00000080:0.0:1769074786.859334:0:63051:0:(genops.c:904:class_export_put()) final put ffff9b0347c8d000/LOV_OSC_UUID 00000020:01000000:0.0:1769074786.859387:0:63051:0:(obd_config.c:2361:class_manual_cleanup()) Manual cleanup of lustre-OST0000-osc-ffff9b0348feb000 (flags='') 00000020:00000080:0.0:1769074786.859393:0:63051:0:(obd_config.c:1389:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1769074786.859406:0:63051:0:(obd_config.c:886:class_cleanup()) lustre-OST0000-osc-ffff9b0348feb000: forcing exports to disconnect: 0/3 00000020:00080000:0.0:1769074786.859411:0:63051:0:(genops.c:1575:class_disconnect_exports()) OBD device 17 (ffff9b0342f16360) has no exports 00000008:01000000:0.0:1769074786.859908:0:63051:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-OST0000-osc-ffff9b0348feb000: client import never connected 00000100:00080000:0.0:1769074786.859912:0:63051:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00100000:0.0:1769074786.859916:0:63051:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9b0348feb000: committing for last_committed 21474841223 gen 3 00000100:00020000:0.0:1769074786.859920:0:63051:0:(obd_class.h:479:obd_check_dev()) Device 17 not setup 00000100:00100000:0.0:1769074786.859923:0:63051:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00020000:0.0:1769074786.859925:0:63051:0:(obd_class.h:479:obd_check_dev()) Device 17 not setup 00000020:00000080:0.0:1769074786.860226:0:63051:0:(obd_config.c:1389:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1769074786.860289:0:63051:0:(obd_config.c:814:class_detach()) detach on obd lustre-OST0000-osc-ffff9b0348feb000 (uuid 0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b) 00000020:00000080:0.0:1769074786.860340:0:63051:0:(genops.c:1485:class_disconnect()) disconnect: cookie 0xb7d6bf6198798ea3 00000020:00000080:0.0:1769074786.860346:0:63051:0:(genops.c:904:class_export_put()) final put ffff9b0347c8f000/0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b 00000100:00080000:0.0:1769074786.860668:0:63051:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b->lustre-MDT0000_UUID 00010000:00010000:0.0:1769074786.861529:0:63051:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b03453e8400/0xb7d6bf6198799f58 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd50b92 expref: -99 pid: 62258 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074786.861592:0:63051:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b03453e8400/0xb7d6bf6198799f58 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd50b92 expref: -99 pid: 62258 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074786.861599:0:63051:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074786.861616:0:63051:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b180 going to sleep for 0 seconds 00000100:00100000:0.0:1769074786.861626:0:63051:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job umount:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:63051:1855007616931840:192.168.204.160@tcp:103:umount.0 00010000:00010000:0.0:1769074786.861636:0:63051:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1769074786.861643:0:63051:0:(ldlm_request.c:2263:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1769074786.861646:0:63051:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b03453e8400/0xb7d6bf6198799f58 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd50b92 expref: -99 pid: 62258 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074786.861788:0:63051:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074786.861807:0:63051:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074786.861814:0:63051:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074786.861820:0:63051:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b180 going to sleep for 0 seconds 00000100:00100000:0.0:1769074786.861824:0:63051:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job umount:0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b:63051:1855007616931968:192.168.204.160@tcp:103:umount.0 00010000:00010000:0.0:1769074786.861830:0:63051:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:0.0:1769074786.861834:0:63051:0:(ldlm_request.c:2263:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:0.0:1769074786.861837:0:63051:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0348feb000 lock: ffff9b0342cd0400/0xb7d6bf6198798eb8 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd4ecf2 expref: -99 pid: 61870 timeout: 0 lvb_type: 0 00000100:00080000:0.0:1769074786.866149:0:63051:0:(import.c:65:import_set_state_nolock()) ffff9b0372746000 lustre-MDT0000_UUID: changing import state from FULL to CONNECTING 00000100:00080000:0.0:1769074786.873052:0:63051:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00100000:0.0:1769074786.873058:0:63051:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0348feb000: committing for last_committed 8589943995 gen 2 00000100:00100000:0.0:1769074786.873071:0:63051:0:(client.c:3119:ptlrpc_free_committed()) @@@ free request with old gen req@ffff9b0365a39180 x1855007616932096/t8589944006(8589944006) o39->lustre-MDT0000-mdc-ffff9b0348feb000@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 0 dl 1769074796 ref 1 fl Complete:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074786.873095:0:63051:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:0.0:1769074786.877053:0:63051:0:(genops.c:1485:class_disconnect()) disconnect: cookie 0xb7d6bf6198798e95 00000020:00000080:0.0:1769074786.877068:0:63051:0:(genops.c:904:class_export_put()) final put ffff9b0342f81800/0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b 00000100:00080000:0.0:1769074786.877320:0:63051:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b->lustre-MDT0001_UUID 00000020:00000080:0.0:1769074786.910981:0:36:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff9b0372747800 for lustre-MDT0001-mdc-ffff9b0348feb000 00000020:00000080:0.0:1769074786.910992:0:36:0:(genops.c:904:class_export_put()) final put ffff9b0372747000/0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b 00000020:00000080:0.0:1769074786.910997:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff9b0372747000/0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b for lustre-MDT0001-mdc-ffff9b0348feb000 00000020:01000000:0.0:1769074786.911011:0:36:0:(genops.c:443:class_free_dev()) finishing cleanup of obd lustre-MDT0001-mdc-ffff9b0348feb000 (0bbd97bd-31b2-46f4-8b1b-d137c4ecd83b) 00000100:00100000:0.0:1769074787.033601:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074787.033641:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074787.033689:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625183232 00000100:00100000:0.0:1769074787.033702:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625183232 00000100:00100000:0.0:1769074787.033761:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625183360 00000100:00100000:0.0:1769074787.033764:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625183360 00000100:00100000:0.0:1769074787.033781:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8862 00000100:00100000:0.0:1769074787.033791:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a39180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+6:8855:x1855007625183232:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074787.033813:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c1fd lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd54417 expref: -99 pid: 63053 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074787.033930:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074787.033938:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a39180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+6:8855:x1855007625183232:12345-192.168.204.160@tcp:104: Request processed in 148us (353us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074787.033947:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8862 00000100:00100000:0.0:1769074787.033969:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8863 00000100:00100000:0.0:1769074787.033976:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a3b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:8855:x1855007625183360:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074787.033984:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c178 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x13/0x0 rrc: 5 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x7fe2323adfd54376 expref: -99 pid: 63007 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074787.034028:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074787.034033:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a3b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:8855:x1855007625183360:12345-192.168.204.160@tcp:104: Request processed in 58us (393us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074787.034040:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8863 00000100:00100000:0.0:1769074787.052815:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625183488 00000100:00100000:0.0:1769074787.052830:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625183488 00000100:00100000:0.0:1769074787.053013:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8864 00000100:00100000:0.0:1769074787.053027:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:8855:x1855007625183488:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074787.053048:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c204 lrc: 2/0,0 mode: CW/CW res: [0x200000405:0x1fb:0x0].0x0 bits 0x5/0x0 rrc: 5 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x7fe2323adfd54425 expref: -99 pid: 63053 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074787.053210:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074787.053219:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:8855:x1855007625183488:12345-192.168.204.160@tcp:104: Request processed in 196us (782us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074787.053227:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8864 00000100:00100000:0.0:1769074787.106300:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625183616 00000100:00100000:0.0:1769074787.106318:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625183616 00000100:00100000:0.0:1769074787.106396:0:55997:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074787.106419:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625183744 00000100:00100000:0.0:1769074787.106424:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625183744 00000100:00100000:0.0:1769074787.106443:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625184000 00000100:00100000:0.0:1769074787.106453:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625184000 00000100:00100000:0.0:1769074787.106490:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8865 00000100:00100000:0.0:1769074787.106510:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cea300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:8398:x1855007625183616:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074787.106529:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c18d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd543ae expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.106636:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074787.106643:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cea300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:8398:x1855007625183616:12345-192.168.204.160@tcp:104: Request processed in 144us (544us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074787.116410:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8865 00000100:00100000:0.0:1769074787.116623:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074787.116808:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625184256 00000100:00100000:0.0:1769074787.116817:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625184256 00000100:00100000:0.0:1769074787.116867:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8869 00000100:00100000:0.0:1769074787.116876:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a39180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:10284:x1855007625184256:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074787.116892:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0344ccc800/0xb7d6bf619879c1b0 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd543a7 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.116959:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:0.0:1769074787.116965:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a39180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:10284:x1855007625184256:12345-192.168.204.160@tcp:104: Request processed in 91us (345us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074787.116971:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8869 00010000:00010000:0.0:1769074787.117020:0:46917:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:0.0:1769074787.117028:0:46917:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c1a2 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd543a0 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.117036:0:46917:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b03466c4e00 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:0.0:1769074787.117043:0:46917:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c1a2 lrc: 3/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800428400000000 nid: local remote: 0x7fe2323adfd543a0 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074787.117102:0:46917:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074787.129906:0:46917:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1769074787.129978:0:46917:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c1a2 lrc: 2/0,0 mode: --/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd543a0 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.129996:0:46917:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074787.130045:0:46917:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c1a2 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd543a0 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.130054:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c1a2 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd543a0 expref: -99 pid: 63007 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.130517:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074787.130519:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074787.130522:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074787.130570:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616934528:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074787.131442:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074787.138122:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074787.138130:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074787.138133:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1769074787.384635:0:63053:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [2048, 2197] 00000008:00010000:0.0:1769074787.384659:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:0.0:1769074787.384664:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df4b0, osc lock ffff9b0345a6c980, flags 0x40000 00000080:00200000:0.0:1769074787.384684:0:63053:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [8388608, 9000032) 00000080:00200000:0.0:1769074787.384698:0:63053:0:(vvp_io.c:886:vvp_io_read_start()) count:611424 iocb pos:8388608 00000080:00200000:0.0:1769074787.384704:0:63053:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), size=611424 (max 603979776), offset=8388608=0x800000, pages 150 (max 147456), lockless, parallel, unaligned 00000020:00200000:0.0:1769074787.384865:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000080:00200000:0.0:1769074787.384868:0:63053:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 611424 count 611424 file_offset 9000032 00000020:00010000:0.0:1769074787.384874:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0343c7e5d8 (ffff9b0353152cf8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074787.384877:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343c7e5d8 (ffff9b0353152cf8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.384880:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df4b0 (ffff9b0342ca8008/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.384883:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343c7e5d8 (ffff9b0353152cf8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.384886:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df4b0 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074787.384892:0:63053:0:(vvp_io.c:518:vvp_io_advance()) advancing 611424 bytes 00000020:00200000:0.0:1769074787.384897:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000008:00010000:0.0:1769074787.384914:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9240 00000008:00100000:0.0:1769074787.384977:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a39f80 - object 0x2c0000401:703 offset 4194304<>4805728 00000100:00100000:0.0:1769074787.385000:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074787.385015:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616938112:192.168.204.160@tcp:3:lfs.0 00000008:00010000:0.0:1769074787.489541:0:2406:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b0347fb6780 00000020:00200000:0.0:1769074787.489555:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1769074787.489559:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1769074787.489561:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1769074787.489562:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1769074787.489564:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1769074787.489565:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1769074787.489567:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1769074787.489569:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1769074787.489570:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1769074787.489571:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1769074787.489573:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1769074787.489575:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1769074787.489576:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1769074787.489577:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1769074787.489579:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1769074787.489580:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1769074787.489582:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1769074787.489583:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1769074787.489585:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1769074787.489586:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1769074787.489588:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1769074787.489589:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1769074787.489590:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1769074787.489592:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1769074787.489594:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1769074787.489595:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1769074787.489596:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1769074787.489598:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1769074787.489606:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1769074787.489608:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1769074787.489609:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1769074787.489611:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1769074787.489612:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1769074787.489614:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1769074787.489615:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1769074787.489617:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1769074787.489618:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1769074787.489620:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1769074787.489622:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1769074787.489623:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1769074787.489625:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1769074787.489627:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1769074787.489628:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1769074787.489630:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1769074787.489631:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1769074787.489633:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1769074787.489634:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1769074787.489635:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1769074787.489637:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1769074787.489638:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1769074787.489639:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1769074787.489641:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1769074787.489643:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1769074787.489644:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1769074787.489646:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1769074787.489647:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1769074787.489649:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1769074787.489651:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1769074787.489652:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1769074787.489654:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1769074787.489655:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1769074787.489656:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1769074787.489658:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1769074787.489659:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1769074787.489661:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1769074787.489663:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1769074787.489934:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1769074787.489936:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1769074787.489938:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1769074787.489939:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1769074787.489941:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1769074787.489942:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1769074787.489943:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1769074787.489945:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1769074787.489946:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1769074787.489947:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1769074787.489948:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1769074787.489950:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1769074787.489951:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1769074787.489953:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1769074787.489955:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1769074787.489956:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1769074787.489958:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1769074787.489960:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1769074787.489962:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1769074787.489963:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1769074787.489964:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1769074787.489966:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1769074787.489968:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1769074787.489970:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1769074787.489971:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1769074787.489973:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1769074787.489974:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1769074787.489975:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1769074787.489977:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1769074787.489978:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1769074787.489980:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1769074787.489981:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1769074787.489982:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1769074787.489984:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1769074787.489986:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1769074787.489987:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1769074787.489989:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1769074787.489990:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1769074787.489992:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1769074787.489994:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1769074787.489995:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1769074787.489997:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1769074787.489998:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1769074787.489999:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1769074787.490001:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1769074787.490002:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1769074787.490004:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1769074787.490005:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1769074787.490007:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1769074787.490008:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1769074787.490009:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1769074787.490011:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1769074787.490012:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1769074787.490014:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1769074787.490015:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1769074787.490017:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1769074787.490018:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1769074787.490019:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1769074787.490021:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1769074787.490022:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1769074787.490024:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1769074787.490025:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1769074787.490026:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1769074787.490028:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1769074787.490029:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1769074787.490031:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1769074787.490032:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1769074787.490034:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1769074787.490035:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1769074787.490037:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1769074787.490038:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1769074787.490040:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1769074787.490041:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1769074787.490043:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1769074787.490044:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1769074787.490045:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1769074787.490047:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1769074787.490049:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1769074787.490050:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1769074787.490052:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1769074787.490053:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1769074787.490054:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1769074787.490056:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1769074787.490058:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1769074787.490059:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1769074787.490062:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1769074787.490063:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1769074787.490066:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1769074787.490067:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1769074787.490069:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1769074787.490070:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1769074787.490072:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1769074787.490073:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1769074787.490074:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1769074787.490076:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1769074787.490077:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1769074787.490079:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1769074787.490080:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1769074787.490082:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1769074787.490083:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1769074787.490084:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1769074787.490086:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1769074787.490088:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1769074787.490089:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1769074787.490090:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1769074787.490092:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1769074787.490093:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1769074787.490095:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1769074787.490096:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1769074787.490097:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1769074787.490099:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1769074787.490100:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1769074787.490102:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1769074787.490104:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1769074787.490105:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1769074787.490107:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1769074787.490108:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1769074787.490110:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1769074787.490112:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1769074787.490114:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1769074787.490115:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1769074787.490117:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1769074787.490118:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1769074787.490119:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1769074787.490121:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1769074787.490122:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1769074787.490124:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1769074787.490125:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1769074787.490127:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1769074787.490129:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1769074787.490130:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1769074787.490132:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1769074787.490133:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1769074787.490135:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1769074787.490136:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1769074787.490138:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1769074787.490140:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1769074787.490142:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1769074787.490143:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1769074787.490145:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1769074787.490146:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1769074787.490148:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1769074787.490149:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1769074787.490151:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1769074787.490153:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1769074787.490220:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1769074787.490222:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1769074787.490223:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1769074787.490225:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1769074787.490227:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1769074787.490229:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1769074787.490230:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1769074787.490258:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1769074787.490260:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1769074787.490262:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1769074787.490264:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1769074787.490265:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1769074787.490267:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1769074787.490269:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1769074787.490270:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1769074787.490272:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1769074787.490274:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1769074787.490276:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1769074787.490277:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1769074787.490279:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1769074787.490280:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1769074787.490281:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1769074787.490283:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1769074787.490284:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1769074787.490285:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1769074787.490287:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1769074787.490288:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1769074787.490289:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1769074787.490290:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1769074787.490291:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1769074787.490293:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1769074787.490294:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1769074787.490295:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1769074787.490297:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1769074787.490298:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1769074787.490299:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1769074787.490301:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1769074787.490302:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1769074787.490303:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1769074787.490305:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1769074787.490306:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1769074787.490308:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1769074787.490309:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1769074787.490311:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1769074787.490312:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1769074787.492572:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1769074787.492584:0:2406:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b0347fb6780 00000100:00100000:0.0:1769074787.492610:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616937984:192.168.204.160@tcp:3:lfs.0 00000008:00010000:0.0:1769074787.500131:0:2407:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b03515b9240 00000020:00200000:0.0:1769074787.500144:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1769074787.500147:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1769074787.500149:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1769074787.500151:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1769074787.500152:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1769074787.500154:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1769074787.500155:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1769074787.500459:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1769074787.500460:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1769074787.500462:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1769074787.500464:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1769074787.500466:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1769074787.500467:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1769074787.500469:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1769074787.500470:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1769074787.500472:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1769074787.500473:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1769074787.500475:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1769074787.500478:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1769074787.500479:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1769074787.500481:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1769074787.500483:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1769074787.500484:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1769074787.500486:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1769074787.500488:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1769074787.500489:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1769074787.500491:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1769074787.500492:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1769074787.500494:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1769074787.500496:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1769074787.500498:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1769074787.500499:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1769074787.500501:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1769074787.500502:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1769074787.500504:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1769074787.500506:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1769074787.500508:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1769074787.500509:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1769074787.500511:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1769074787.500512:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1769074787.500514:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1769074787.500516:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1769074787.500518:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1769074787.500519:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1769074787.500521:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1769074787.500522:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1769074787.500524:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1769074787.500525:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1769074787.500527:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1769074787.500528:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1769074787.500530:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1769074787.500532:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1769074787.500534:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1769074787.500536:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1769074787.500537:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1769074787.500539:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1769074787.500541:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1769074787.500542:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1769074787.500544:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1769074787.500546:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1769074787.500547:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1769074787.500548:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1769074787.500550:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1769074787.500552:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1769074787.500554:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1769074787.500555:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1769074787.500557:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1769074787.500558:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1769074787.500560:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1769074787.500562:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1769074787.500563:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1769074787.500565:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1769074787.500567:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1769074787.500568:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1769074787.500570:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1769074787.500572:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1769074787.500574:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1769074787.500575:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1769074787.500576:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1769074787.500578:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1769074787.500579:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1769074787.500581:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1769074787.500583:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1769074787.500585:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1769074787.500586:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1769074787.500588:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1769074787.500589:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1769074787.500591:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1769074787.500593:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1769074787.500594:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1769074787.500596:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1769074787.500613:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1769074787.500616:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1769074787.500617:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1769074787.500619:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1769074787.500621:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1769074787.500622:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1769074787.500624:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1769074787.500625:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1769074787.500627:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1769074787.500628:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1769074787.500630:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1769074787.500631:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1769074787.500633:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1769074787.500634:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1769074787.500636:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1769074787.500638:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1769074787.500640:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1769074787.500642:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1769074787.500643:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1769074787.500645:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1769074787.500647:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1769074787.500648:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1769074787.500650:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1769074787.500652:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1769074787.500654:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1769074787.500656:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1769074787.500657:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1769074787.500659:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1769074787.500660:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1769074787.500728:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1769074787.500729:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1769074787.500731:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1769074787.500732:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1769074787.500734:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1769074787.500735:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1769074787.500737:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1769074787.500739:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1769074787.500740:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1769074787.500742:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1769074787.500743:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1769074787.500745:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1769074787.500746:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1769074787.500747:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1769074787.500749:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1769074787.500750:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1769074787.500751:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1769074787.500753:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1769074787.500755:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1769074787.500756:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1769074787.500758:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1769074787.500759:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1769074787.500761:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1769074787.500763:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1769074787.500764:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1769074787.500766:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1769074787.500768:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1769074787.500769:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1769074787.500771:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1769074787.500773:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1769074787.500838:0:2407:0:(cl_io.c:1316:cl_sub_dio_end()) finishing unaligned dio read aio->cda_bytes 611424 00000020:00200000:0.0:1769074787.500855:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 611424, offset 0, pos 8388608, cdp_page_count 150 00000020:00200000:0.0:1769074787.500862:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 607328, offset 0, pos 8392704, cdp_page_count 150 00000020:00200000:0.0:1769074787.500867:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 603232, offset 0, pos 8396800, cdp_page_count 150 00000020:00200000:0.0:1769074787.500871:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 599136, offset 0, pos 8400896, cdp_page_count 150 00000020:00200000:0.0:1769074787.500875:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 595040, offset 0, pos 8404992, cdp_page_count 150 00000020:00200000:0.0:1769074787.500879:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 590944, offset 0, pos 8409088, cdp_page_count 150 00000020:00200000:0.0:1769074787.500882:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 586848, offset 0, pos 8413184, cdp_page_count 150 00000020:00200000:0.0:1769074787.500886:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 582752, offset 0, pos 8417280, cdp_page_count 150 00000020:00200000:0.0:1769074787.500889:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 578656, offset 0, pos 8421376, cdp_page_count 150 00000020:00200000:0.0:1769074787.500893:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 574560, offset 0, pos 8425472, cdp_page_count 150 00000020:00200000:0.0:1769074787.500896:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 570464, offset 0, pos 8429568, cdp_page_count 150 00000020:00200000:0.0:1769074787.500901:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 566368, offset 0, pos 8433664, cdp_page_count 150 00000020:00200000:0.0:1769074787.500905:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 562272, offset 0, pos 8437760, cdp_page_count 150 00000020:00200000:0.0:1769074787.500909:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 558176, offset 0, pos 8441856, cdp_page_count 150 00000020:00200000:0.0:1769074787.500913:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 554080, offset 0, pos 8445952, cdp_page_count 150 00000020:00200000:0.0:1769074787.500917:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 549984, offset 0, pos 8450048, cdp_page_count 150 00000020:00200000:0.0:1769074787.500921:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 545888, offset 0, pos 8454144, cdp_page_count 150 00000020:00200000:0.0:1769074787.500925:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 541792, offset 0, pos 8458240, cdp_page_count 150 00000020:00200000:0.0:1769074787.500928:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 537696, offset 0, pos 8462336, cdp_page_count 150 00000020:00200000:0.0:1769074787.500932:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 533600, offset 0, pos 8466432, cdp_page_count 150 00000020:00200000:0.0:1769074787.500936:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 529504, offset 0, pos 8470528, cdp_page_count 150 00000020:00200000:0.0:1769074787.500940:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 525408, offset 0, pos 8474624, cdp_page_count 150 00000020:00200000:0.0:1769074787.500944:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 521312, offset 0, pos 8478720, cdp_page_count 150 00000020:00200000:0.0:1769074787.500948:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 517216, offset 0, pos 8482816, cdp_page_count 150 00000020:00200000:0.0:1769074787.500951:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 513120, offset 0, pos 8486912, cdp_page_count 150 00000020:00200000:0.0:1769074787.500954:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 509024, offset 0, pos 8491008, cdp_page_count 150 00000020:00200000:0.0:1769074787.500958:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 504928, offset 0, pos 8495104, cdp_page_count 150 00000020:00200000:0.0:1769074787.500962:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 500832, offset 0, pos 8499200, cdp_page_count 150 00000020:00200000:0.0:1769074787.500966:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 496736, offset 0, pos 8503296, cdp_page_count 150 00000020:00200000:0.0:1769074787.500971:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 492640, offset 0, pos 8507392, cdp_page_count 150 00000020:00200000:0.0:1769074787.500974:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 488544, offset 0, pos 8511488, cdp_page_count 150 00000020:00200000:0.0:1769074787.500978:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 484448, offset 0, pos 8515584, cdp_page_count 150 00000020:00200000:0.0:1769074787.500981:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 480352, offset 0, pos 8519680, cdp_page_count 150 00000020:00200000:0.0:1769074787.500984:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 476256, offset 0, pos 8523776, cdp_page_count 150 00000020:00200000:0.0:1769074787.500987:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 472160, offset 0, pos 8527872, cdp_page_count 150 00000020:00200000:0.0:1769074787.500990:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 468064, offset 0, pos 8531968, cdp_page_count 150 00000020:00200000:0.0:1769074787.500993:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 463968, offset 0, pos 8536064, cdp_page_count 150 00000020:00200000:0.0:1769074787.500996:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 459872, offset 0, pos 8540160, cdp_page_count 150 00000020:00200000:0.0:1769074787.500999:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 455776, offset 0, pos 8544256, cdp_page_count 150 00000020:00200000:0.0:1769074787.501003:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 451680, offset 0, pos 8548352, cdp_page_count 150 00000020:00200000:0.0:1769074787.501007:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 447584, offset 0, pos 8552448, cdp_page_count 150 00000020:00200000:0.0:1769074787.501010:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 443488, offset 0, pos 8556544, cdp_page_count 150 00000020:00200000:0.0:1769074787.501014:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 439392, offset 0, pos 8560640, cdp_page_count 150 00000020:00200000:0.0:1769074787.501018:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 435296, offset 0, pos 8564736, cdp_page_count 150 00000020:00200000:0.0:1769074787.501021:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 431200, offset 0, pos 8568832, cdp_page_count 150 00000020:00200000:0.0:1769074787.501024:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 427104, offset 0, pos 8572928, cdp_page_count 150 00000020:00200000:0.0:1769074787.501028:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 423008, offset 0, pos 8577024, cdp_page_count 150 00000020:00200000:0.0:1769074787.501031:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 418912, offset 0, pos 8581120, cdp_page_count 150 00000020:00200000:0.0:1769074787.501035:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 414816, offset 0, pos 8585216, cdp_page_count 150 00000020:00200000:0.0:1769074787.501038:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 410720, offset 0, pos 8589312, cdp_page_count 150 00000020:00200000:0.0:1769074787.501042:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 406624, offset 0, pos 8593408, cdp_page_count 150 00000020:00200000:0.0:1769074787.501046:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 402528, offset 0, pos 8597504, cdp_page_count 150 00000020:00200000:0.0:1769074787.501050:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 398432, offset 0, pos 8601600, cdp_page_count 150 00000020:00200000:0.0:1769074787.501054:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 394336, offset 0, pos 8605696, cdp_page_count 150 00000020:00200000:0.0:1769074787.501058:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 390240, offset 0, pos 8609792, cdp_page_count 150 00000020:00200000:0.0:1769074787.501061:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 386144, offset 0, pos 8613888, cdp_page_count 150 00000020:00200000:0.0:1769074787.501065:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 382048, offset 0, pos 8617984, cdp_page_count 150 00000020:00200000:0.0:1769074787.501069:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 377952, offset 0, pos 8622080, cdp_page_count 150 00000020:00200000:0.0:1769074787.501072:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 373856, offset 0, pos 8626176, cdp_page_count 150 00000020:00200000:0.0:1769074787.501075:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 369760, offset 0, pos 8630272, cdp_page_count 150 00000020:00200000:0.0:1769074787.501078:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 365664, offset 0, pos 8634368, cdp_page_count 150 00000020:00200000:0.0:1769074787.501082:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 361568, offset 0, pos 8638464, cdp_page_count 150 00000020:00200000:0.0:1769074787.501085:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 357472, offset 0, pos 8642560, cdp_page_count 150 00000020:00200000:0.0:1769074787.501088:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 353376, offset 0, pos 8646656, cdp_page_count 150 00000020:00200000:0.0:1769074787.501091:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 349280, offset 0, pos 8650752, cdp_page_count 150 00000020:00200000:0.0:1769074787.501094:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 345184, offset 0, pos 8654848, cdp_page_count 150 00000020:00200000:0.0:1769074787.501097:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 341088, offset 0, pos 8658944, cdp_page_count 150 00000020:00200000:0.0:1769074787.501100:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 336992, offset 0, pos 8663040, cdp_page_count 150 00000020:00200000:0.0:1769074787.501103:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 332896, offset 0, pos 8667136, cdp_page_count 150 00000020:00200000:0.0:1769074787.501106:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 328800, offset 0, pos 8671232, cdp_page_count 150 00000020:00200000:0.0:1769074787.501109:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 324704, offset 0, pos 8675328, cdp_page_count 150 00000020:00200000:0.0:1769074787.501112:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 320608, offset 0, pos 8679424, cdp_page_count 150 00000020:00200000:0.0:1769074787.501115:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 316512, offset 0, pos 8683520, cdp_page_count 150 00000020:00200000:0.0:1769074787.501118:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 312416, offset 0, pos 8687616, cdp_page_count 150 00000020:00200000:0.0:1769074787.501122:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 308320, offset 0, pos 8691712, cdp_page_count 150 00000020:00200000:0.0:1769074787.501126:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 304224, offset 0, pos 8695808, cdp_page_count 150 00000020:00200000:0.0:1769074787.501130:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 300128, offset 0, pos 8699904, cdp_page_count 150 00000020:00200000:0.0:1769074787.501134:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 296032, offset 0, pos 8704000, cdp_page_count 150 00000020:00200000:0.0:1769074787.501138:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 291936, offset 0, pos 8708096, cdp_page_count 150 00000020:00200000:0.0:1769074787.501142:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 287840, offset 0, pos 8712192, cdp_page_count 150 00000020:00200000:0.0:1769074787.501146:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 283744, offset 0, pos 8716288, cdp_page_count 150 00000020:00200000:0.0:1769074787.501150:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 279648, offset 0, pos 8720384, cdp_page_count 150 00000020:00200000:0.0:1769074787.501153:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 275552, offset 0, pos 8724480, cdp_page_count 150 00000020:00200000:0.0:1769074787.501491:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 271456, offset 0, pos 8728576, cdp_page_count 150 00000020:00200000:0.0:1769074787.501497:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 267360, offset 0, pos 8732672, cdp_page_count 150 00000020:00200000:0.0:1769074787.501501:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 263264, offset 0, pos 8736768, cdp_page_count 150 00000020:00200000:0.0:1769074787.501505:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 259168, offset 0, pos 8740864, cdp_page_count 150 00000020:00200000:0.0:1769074787.501508:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 255072, offset 0, pos 8744960, cdp_page_count 150 00000020:00200000:0.0:1769074787.501512:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 250976, offset 0, pos 8749056, cdp_page_count 150 00000020:00200000:0.0:1769074787.501516:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 246880, offset 0, pos 8753152, cdp_page_count 150 00000020:00200000:0.0:1769074787.501518:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 242784, offset 0, pos 8757248, cdp_page_count 150 00000020:00200000:0.0:1769074787.501522:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 238688, offset 0, pos 8761344, cdp_page_count 150 00000020:00200000:0.0:1769074787.501525:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 234592, offset 0, pos 8765440, cdp_page_count 150 00000020:00200000:0.0:1769074787.501529:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 230496, offset 0, pos 8769536, cdp_page_count 150 00000020:00200000:0.0:1769074787.501532:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 226400, offset 0, pos 8773632, cdp_page_count 150 00000020:00200000:0.0:1769074787.501535:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 222304, offset 0, pos 8777728, cdp_page_count 150 00000020:00200000:0.0:1769074787.501539:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 218208, offset 0, pos 8781824, cdp_page_count 150 00000020:00200000:0.0:1769074787.501542:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 214112, offset 0, pos 8785920, cdp_page_count 150 00000020:00200000:0.0:1769074787.501546:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 210016, offset 0, pos 8790016, cdp_page_count 150 00000020:00200000:0.0:1769074787.501550:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 205920, offset 0, pos 8794112, cdp_page_count 150 00000020:00200000:0.0:1769074787.501553:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 201824, offset 0, pos 8798208, cdp_page_count 150 00000020:00200000:0.0:1769074787.501556:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 197728, offset 0, pos 8802304, cdp_page_count 150 00000020:00200000:0.0:1769074787.501560:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 193632, offset 0, pos 8806400, cdp_page_count 150 00000020:00200000:0.0:1769074787.501563:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 189536, offset 0, pos 8810496, cdp_page_count 150 00000020:00200000:0.0:1769074787.501567:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 185440, offset 0, pos 8814592, cdp_page_count 150 00000020:00200000:0.0:1769074787.501570:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 181344, offset 0, pos 8818688, cdp_page_count 150 00000020:00200000:0.0:1769074787.501574:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 177248, offset 0, pos 8822784, cdp_page_count 150 00000020:00200000:0.0:1769074787.501577:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 173152, offset 0, pos 8826880, cdp_page_count 150 00000020:00200000:0.0:1769074787.501580:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 169056, offset 0, pos 8830976, cdp_page_count 150 00000020:00200000:0.0:1769074787.501584:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 164960, offset 0, pos 8835072, cdp_page_count 150 00000020:00200000:0.0:1769074787.501587:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 160864, offset 0, pos 8839168, cdp_page_count 150 00000020:00200000:0.0:1769074787.501590:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 156768, offset 0, pos 8843264, cdp_page_count 150 00000020:00200000:0.0:1769074787.501593:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 152672, offset 0, pos 8847360, cdp_page_count 150 00000020:00200000:0.0:1769074787.501596:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 148576, offset 0, pos 8851456, cdp_page_count 150 00000020:00200000:0.0:1769074787.501599:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 144480, offset 0, pos 8855552, cdp_page_count 150 00000020:00200000:0.0:1769074787.501601:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 140384, offset 0, pos 8859648, cdp_page_count 150 00000020:00200000:0.0:1769074787.501604:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 136288, offset 0, pos 8863744, cdp_page_count 150 00000020:00200000:0.0:1769074787.501607:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 132192, offset 0, pos 8867840, cdp_page_count 150 00000020:00200000:0.0:1769074787.501610:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 128096, offset 0, pos 8871936, cdp_page_count 150 00000020:00200000:0.0:1769074787.501613:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 124000, offset 0, pos 8876032, cdp_page_count 150 00000020:00200000:0.0:1769074787.501616:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 119904, offset 0, pos 8880128, cdp_page_count 150 00000020:00200000:0.0:1769074787.501619:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 115808, offset 0, pos 8884224, cdp_page_count 150 00000020:00200000:0.0:1769074787.501621:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 111712, offset 0, pos 8888320, cdp_page_count 150 00000020:00200000:0.0:1769074787.501624:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 107616, offset 0, pos 8892416, cdp_page_count 150 00000020:00200000:0.0:1769074787.501628:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 103520, offset 0, pos 8896512, cdp_page_count 150 00000020:00200000:0.0:1769074787.501631:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 99424, offset 0, pos 8900608, cdp_page_count 150 00000020:00200000:0.0:1769074787.501634:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 95328, offset 0, pos 8904704, cdp_page_count 150 00000020:00200000:0.0:1769074787.501637:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 91232, offset 0, pos 8908800, cdp_page_count 150 00000020:00200000:0.0:1769074787.501640:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 87136, offset 0, pos 8912896, cdp_page_count 150 00000020:00200000:0.0:1769074787.501643:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 83040, offset 0, pos 8916992, cdp_page_count 150 00000020:00200000:0.0:1769074787.501647:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 78944, offset 0, pos 8921088, cdp_page_count 150 00000020:00200000:0.0:1769074787.501651:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 74848, offset 0, pos 8925184, cdp_page_count 150 00000020:00200000:0.0:1769074787.501654:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 70752, offset 0, pos 8929280, cdp_page_count 150 00000020:00200000:0.0:1769074787.501657:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 66656, offset 0, pos 8933376, cdp_page_count 150 00000020:00200000:0.0:1769074787.501660:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 62560, offset 0, pos 8937472, cdp_page_count 150 00000020:00200000:0.0:1769074787.501663:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 58464, offset 0, pos 8941568, cdp_page_count 150 00000020:00200000:0.0:1769074787.502278:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 54368, offset 0, pos 8945664, cdp_page_count 150 00000020:00200000:0.0:1769074787.502283:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 50272, offset 0, pos 8949760, cdp_page_count 150 00000020:00200000:0.0:1769074787.502287:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 46176, offset 0, pos 8953856, cdp_page_count 150 00000020:00200000:0.0:1769074787.502291:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 42080, offset 0, pos 8957952, cdp_page_count 150 00000020:00200000:0.0:1769074787.502295:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 37984, offset 0, pos 8962048, cdp_page_count 150 00000020:00200000:0.0:1769074787.502299:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 33888, offset 0, pos 8966144, cdp_page_count 150 00000020:00200000:0.0:1769074787.502303:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 29792, offset 0, pos 8970240, cdp_page_count 150 00000020:00200000:0.0:1769074787.502306:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 25696, offset 0, pos 8974336, cdp_page_count 150 00000020:00200000:0.0:1769074787.502310:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 21600, offset 0, pos 8978432, cdp_page_count 150 00000020:00200000:0.0:1769074787.502313:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 17504, offset 0, pos 8982528, cdp_page_count 150 00000020:00200000:0.0:1769074787.502317:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 13408, offset 0, pos 8986624, cdp_page_count 150 00000020:00200000:0.0:1769074787.502321:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 9312, offset 0, pos 8990720, cdp_page_count 150 00000020:00200000:0.0:1769074787.502420:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 5216, offset 0, pos 8994816, cdp_page_count 150 00000020:00200000:0.0:1769074787.502424:0:2407:0:(cl_io.c:1643:__ll_dio_user_copy()) count 1120, offset 0, pos 8998912, cdp_page_count 150 00000020:00200000:0.0:1769074787.502444:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1769074787.502475:0:2407:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b03515b9240 00000100:00100000:0.0:1769074787.502514:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616938112:192.168.204.160@tcp:3:lfs.0 00000080:00200000:0.0:1769074787.643468:0:63053:0:(file.c:2109:ll_file_io_generic()) Range unlock [1536, 2047] 00000080:00200000:0.0:1769074787.643489:0:63053:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 31 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074787.643494:0:63053:0:(file.c:2142:ll_file_io_generic()) f200b.sanity-flr: 2 io complete with rc: 0, result: 2097152, restart: 0 00000080:00200000:0.0:1769074787.643509:0:63053:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 17576 00000020:00200000:0.0:1769074787.643513:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074787.643521:0:63053:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 2097152 00000080:00200080:0.0:1769074787.643541:0:63053:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8) cmd=400866b1 arg=0 00000080:00200080:0.0:1769074787.643552:0:63053:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8) cmd=400866b1 arg=2 00000080:00200008:0.0:1769074787.643559:0:63053:0:(file.c:6252:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074787.643566:0:63053:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8),name=f200b.sanity-flr 00000002:00010000:0.0:1769074787.643589:0:63053:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x1fb:0x0]) in obj [0x200000405:0x1fb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074787.643623:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c220 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54495 expref: -99 pid: 63053 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074787.643638:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c220 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54495 expref: -99 pid: 63053 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074787.643647:0:63053:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000405:0x1fb:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074787.643655:0:63053:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x1fb:0x0](ffff9b0355ce63c8) 00000080:00010000:0.0:1769074787.643658:0:63053:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x1fb:0x0](ffff9b0355ce63c8) for lock 0xb7d6bf619879c220 00000080:00010000:0.0:1769074787.643983:0:63053:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c220 from it ffffb43642f9fcd8 00010000:00010000:0.0:1769074787.643989:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c220 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54495 expref: -99 pid: 63053 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074787.643997:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c220 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54495 expref: -99 pid: 63053 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074787.644027:0:63053:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 31 restore needed 0 00020000:00200000:0.0:1769074787.644040:0:63053:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074787.644044:0:63053:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1769074787.644048:0:63053:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fb:0x0] 00000008:00010000:0.0:1769074787.644071:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.644075:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902c30, osc lock ffff9b0345a6c980, flags 0x400000400001002 00000008:00010000:0.0:1769074787.644082:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.644084:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902c70, osc lock ffff9b0345a6cf70, flags 0x400000400001002 00000008:00010000:0.0:1769074787.644089:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.644091:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902cb0, osc lock ffff9b0345a6dc80, flags 0x400000400001002 00000008:00010000:0.0:1769074787.644096:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.644097:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902cf0, osc lock ffff9b0345a6d7c0, flags 0x400000400001002 00000008:00010000:0.0:1769074787.644127:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.644129:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902d30, osc lock ffff9b0345a6da20, flags 0x400000400001002 00000008:00010000:0.0:1769074787.644145:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.644147:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902d70, osc lock ffff9b0345a6c390, flags 0x400000400001002 00010000:00010000:0.0:1769074787.644296:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c227 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5449c expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644310:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c227 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5449c expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.644415:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.644420:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c227 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5449c expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644429:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c227 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5449c expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644451:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341219e00/0xb7d6bf619879c22e lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b1 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644460:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341219e00/0xb7d6bf619879c22e lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b1 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.644467:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.644470:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341219e00/0xb7d6bf619879c22e lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd544b1 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644477:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341219e00/0xb7d6bf619879c22e lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b1 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644489:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341218400/0xb7d6bf619879c235 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544a3 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644497:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341218400/0xb7d6bf619879c235 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544a3 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.644503:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.644506:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341218400/0xb7d6bf619879c235 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd544a3 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644513:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341218400/0xb7d6bf619879c235 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544a3 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644528:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341219c00/0xb7d6bf619879c23c lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544bf expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644535:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341219c00/0xb7d6bf619879c23c lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544bf expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.644542:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.644544:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341219c00/0xb7d6bf619879c23c lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd544bf expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644551:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341219c00/0xb7d6bf619879c23c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544bf expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644561:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341218a00/0xb7d6bf619879c243 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544aa expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644570:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 8388607) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341218a00/0xb7d6bf619879c243 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544aa expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.644577:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.644579:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341218a00/0xb7d6bf619879c243 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd544aa expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644586:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341218a00/0xb7d6bf619879c243 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544aa expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644824:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c24a lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b8 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644835:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 8388607) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c24a lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b8 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.644843:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.644855:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c24a lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd544b8 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.644862:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c24a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b8 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.644870:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074787.644882:0:63053:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 17576 00000020:00010000:0.0:1769074787.644886:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314f478 (ffff9b0353152cf8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074787.644888:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314f478 (ffff9b0353152cf8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.644892:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902c30 (ffff9b03531526c8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.644896:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902c70 (ffff9b0342ca8518/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.644899:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902cb0 (ffff9b0342b7d518/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.644901:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902cf0 (ffff9b0342b7d5a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.644903:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902d30 (ffff9b0342b7d2d8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.644906:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902d70 (ffff9b0342b7d758/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.644908:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314f478 (ffff9b0353152cf8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.644911:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902c30 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.644918:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902c70 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.644921:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902cb0 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.644924:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902cf0 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.644926:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902d30 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.644928:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902d70 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074787.644946:0:63053:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 31 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1769074787.644972:0:63053:0:(file.c:2648:do_file_write_iter()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0], ppos: 8388608, count: 614400 00000080:00200000:0.0:1769074787.644981:0:63053:0:(file.c:1994:ll_file_io_generic()) f200b.sanity-flr: write ppos: 8388608, bytes: 614400 00000080:00200000:0.0:1769074787.644992:0:63053:0:(file.c:1759:ll_io_set_mirror()) f200b.sanity-flr: desiginated mirror: 2 00000020:00200000:0.0:1769074787.645001:0:63053:0:(cl_io.c:214:cl_io_rw_init()) header@0000000006fe93a2[0x4, 2, [0x200000405:0x1fb:0x0] hash] 00000020:00200000:0.0:1769074787.645003:0:63053:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [8388608, 9003008) 2048 0 00000080:00200000:0.0:1769074787.645006:0:63053:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 31 restore needed 0 00000080:00200000:0.0:1769074787.645020:0:63053:0:(file.c:2072:ll_file_io_generic()) Range lock [2048, 2197] 00020000:00200000:0.0:1769074787.645029:0:63053:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 8 chunk: [8388608, 9003008) 9003008, 614400 00020000:00200000:0.0:1769074787.645033:0:63053:0:(lov_io.c:860:lov_io_iter_init()) component[8] flags 0x11 00000080:00200000:0.0:1769074787.645049:0:63053:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [2048, 2197] 00000008:00010000:0.0:1769074787.645058:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:0.0:1769074787.645061:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df4b0, osc lock ffff9b0345a6c390, flags 0x40000 00000080:00200000:0.0:1769074787.645070:0:63053:0:(vvp_io.c:1312:vvp_io_write_start()) f200b.sanity-flr: write [8388608, 9003008) append 0 append lockpos 0 00000080:00200000:0.0:1769074787.645083:0:63053:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), size=614400 (max 603979776), offset=8388608=0x800000, pages 150 (max 147456), lockless, parallel 00000020:00200000:0.0:1769074787.645466:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000080:00200000:0.0:1769074787.645469:0:63053:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 614400 count 614400 file_offset 9003008 00000080:00200000:0.0:1769074787.645482:0:63053:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 17576 00000020:00010000:0.0:1769074787.645486:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0343c7e5d8 (ffff9b0353152cf8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074787.645488:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343c7e5d8 (ffff9b0353152cf8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.645491:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df4b0 (ffff9b0342b7d758/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.645493:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343c7e5d8 (ffff9b0353152cf8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.645495:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df4b0 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074787.645500:0:63053:0:(vvp_io.c:518:vvp_io_advance()) advancing 614400 bytes 00000020:00200000:0.0:1769074787.645506:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1769074787.645530:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9840 00000008:00100000:0.0:1769074787.646644:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a39f80 - object 0x280000401:703 offset 4194304<>4808704 00000100:00100000:0.0:1769074787.664503:0:2407:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9b0342ad2800: committing for last_committed 21474841251 gen 1 00000008:00010000:0.0:1769074787.664555:0:2407:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b03515b9840 00000020:00200000:0.0:1769074787.664566:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1769074787.664569:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1769074787.664571:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1769074787.664573:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1769074787.664575:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1769074787.664576:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1769074787.664577:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1769074787.664579:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1769074787.664581:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1769074787.664583:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1769074787.664585:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1769074787.664586:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1769074787.664588:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1769074787.664590:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1769074787.664591:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1769074787.664593:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1769074787.664594:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1769074787.664596:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1769074787.664598:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1769074787.664599:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1769074787.664601:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1769074787.664603:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1769074787.664604:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1769074787.664606:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1769074787.664607:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1769074787.664609:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1769074787.664610:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1769074787.664612:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1769074787.664614:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1769074787.664616:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1769074787.664617:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1769074787.664619:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1769074787.664620:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1769074787.664622:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1769074787.664624:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1769074787.664625:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1769074787.664626:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1769074787.664628:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1769074787.664629:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1769074787.664630:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1769074787.664632:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1769074787.664633:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1769074787.664635:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1769074787.664636:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1769074787.664638:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1769074787.664640:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1769074787.664641:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1769074787.664643:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1769074787.664644:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1769074787.664646:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1769074787.664647:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1769074787.664649:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1769074787.664650:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1769074787.664652:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1769074787.664653:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1769074787.664655:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1769074787.664657:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1769074787.664659:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1769074787.664661:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1769074787.664663:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1769074787.664665:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1769074787.664666:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1769074787.664668:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1769074787.664669:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1769074787.664671:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1769074787.664672:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1769074787.664673:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1769074787.664675:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1769074787.664677:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1769074787.664678:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1769074787.664679:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1769074787.664681:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1769074787.664683:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1769074787.664684:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1769074787.664686:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1769074787.664688:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1769074787.664689:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1769074787.664691:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1769074787.664692:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1769074787.664694:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1769074787.664695:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1769074787.664696:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1769074787.664698:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1769074787.664699:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1769074787.664701:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1769074787.664702:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1769074787.664704:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1769074787.664705:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1769074787.664707:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1769074787.664709:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1769074787.664710:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1769074787.664712:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1769074787.664714:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1769074787.664716:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1769074787.664717:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1769074787.664719:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1769074787.664720:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1769074787.664722:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1769074787.664723:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1769074787.664725:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1769074787.664726:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1769074787.664727:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1769074787.664729:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1769074787.664730:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1769074787.664732:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1769074787.664734:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1769074787.664735:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1769074787.664736:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1769074787.664738:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1769074787.664740:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1769074787.664741:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1769074787.664743:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1769074787.664744:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1769074787.664746:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1769074787.664747:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1769074787.664749:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1769074787.664750:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1769074787.664752:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1769074787.664754:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1769074787.664755:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1769074787.664757:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1769074787.664758:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1769074787.664760:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1769074787.664761:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1769074787.664763:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1769074787.664765:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1769074787.664767:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1769074787.664768:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1769074787.664770:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1769074787.664772:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1769074787.664773:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1769074787.664775:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1769074787.664776:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1769074787.664778:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1769074787.664780:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1769074787.664782:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1769074787.664784:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1769074787.664786:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1769074787.664787:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1769074787.664789:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1769074787.664790:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1769074787.664792:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1769074787.664793:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1769074787.664795:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1769074787.664797:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1769074787.664798:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1769074787.664800:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1769074787.664802:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1769074787.664803:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1769074787.664805:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1769074787.664874:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1769074787.664901:0:2407:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b03515b9840 00000100:00100000:0.0:1769074787.664927:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616939008:192.168.204.160@tcp:4:lfs.0 00000080:00200000:0.0:1769074787.742925:0:63053:0:(file.c:2109:ll_file_io_generic()) Range unlock [1024, 2047] 00000080:00200000:0.0:1769074787.742950:0:63053:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 31 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074787.742956:0:63053:0:(file.c:2142:ll_file_io_generic()) f200b.sanity-flr: 2 io complete with rc: 0, result: 4194304, restart: 0 00000080:00200000:0.0:1769074787.742974:0:63053:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 17576 00000020:00200000:0.0:1769074787.742979:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074787.742993:0:63053:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 4194304 00000080:00200080:0.0:1769074787.743113:0:63053:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8) cmd=400866b1 arg=0 00000080:00200080:0.0:1769074787.743125:0:63053:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8) cmd=400866b1 arg=3 00000080:00200008:0.0:1769074787.743132:0:63053:0:(file.c:6252:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074787.743138:0:63053:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8),name=f200b.sanity-flr 00000002:00010000:0.0:1769074787.743163:0:63053:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x1fb:0x0]) in obj [0x200000405:0x1fb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074787.743181:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c220 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54495 expref: -99 pid: 63053 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074787.743197:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c220 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54495 expref: -99 pid: 63053 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074787.743205:0:63053:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000405:0x1fb:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074787.743212:0:63053:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x1fb:0x0](ffff9b0355ce63c8) 00000080:00010000:0.0:1769074787.743216:0:63053:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x1fb:0x0](ffff9b0355ce63c8) for lock 0xb7d6bf619879c220 00000080:00010000:0.0:1769074787.743223:0:63053:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c220 from it ffffb43642f9fcd8 00010000:00010000:0.0:1769074787.743227:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c220 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54495 expref: -99 pid: 63053 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074787.743324:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c220 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54495 expref: -99 pid: 63053 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074787.743360:0:63053:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 31 restore needed 0 00020000:00200000:0.0:1769074787.743370:0:63053:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074787.743374:0:63053:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1769074787.743378:0:63053:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fb:0x0] 00000008:00010000:0.0:1769074787.743398:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.743401:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0372edde30, osc lock ffff9b0345a6c390, flags 0x400000400001002 00000008:00010000:0.0:1769074787.743408:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.743410:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0372edde70, osc lock ffff9b0345a6da20, flags 0x400000400001002 00000008:00010000:0.0:1769074787.743414:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.743416:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0372eddeb0, osc lock ffff9b0345a6d7c0, flags 0x400000400001002 00000008:00010000:0.0:1769074787.743421:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.743422:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0372eddef0, osc lock ffff9b0345a6dc80, flags 0x400000400001002 00000008:00010000:0.0:1769074787.743427:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.743429:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0372eddf30, osc lock ffff9b0345a6cf70, flags 0x400000400001002 00000008:00010000:0.0:1769074787.743435:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074787.743436:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0372eddf70, osc lock ffff9b0345a6c980, flags 0x400000400001002 00010000:00010000:0.0:1769074787.743453:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c227 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5449c expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743463:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c227 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5449c expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.743473:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.743477:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c227 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5449c expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743484:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c227 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5449c expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743500:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341219e00/0xb7d6bf619879c22e lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b1 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743508:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341219e00/0xb7d6bf619879c22e lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b1 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.743516:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.743519:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341219e00/0xb7d6bf619879c22e lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd544b1 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743527:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341219e00/0xb7d6bf619879c22e lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b1 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743540:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341218400/0xb7d6bf619879c235 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544a3 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743548:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341218400/0xb7d6bf619879c235 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544a3 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.743556:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.743558:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341218400/0xb7d6bf619879c235 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd544a3 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743565:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341218400/0xb7d6bf619879c235 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544a3 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743577:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341219c00/0xb7d6bf619879c23c lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544bf expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743584:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341219c00/0xb7d6bf619879c23c lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544bf expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.743596:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.743599:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341219c00/0xb7d6bf619879c23c lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd544bf expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743606:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341219c00/0xb7d6bf619879c23c lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544bf expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743617:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341218a00/0xb7d6bf619879c243 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544aa expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743624:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 8388607) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341218a00/0xb7d6bf619879c243 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544aa expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.743632:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.743634:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341218a00/0xb7d6bf619879c243 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd544aa expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743642:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341218a00/0xb7d6bf619879c243 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544aa expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.743653:0:63053:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c24a lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b8 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.744001:0:63053:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 8388607) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c24a lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b8 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.744137:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074787.744141:0:63053:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c24a lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd544b8 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.744149:0:63053:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c24a lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd544b8 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074787.744159:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074787.744171:0:63053:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 17576 00000020:00010000:0.0:1769074787.744175:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314f478 (ffff9b0353152cf8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074787.744178:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314f478 (ffff9b0353152cf8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.744182:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0372edde30 (ffff9b0342ca8f38/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.744186:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0372edde70 (ffff9b0342b7d2d8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.744189:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0372eddeb0 (ffff9b0342b7d5a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.744191:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0372eddef0 (ffff9b0342b7d518/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.744193:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0372eddf30 (ffff9b0342ca8518/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.744196:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0372eddf70 (ffff9b03531526c8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.744198:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314f478 (ffff9b0353152cf8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.744201:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0372edde30 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.744205:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0372edde70 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.744208:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0372eddeb0 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.744210:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0372eddef0 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.744213:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0372eddf30 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.744215:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0372eddf70 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074787.744227:0:63053:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 31 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1769074787.744329:0:63053:0:(file.c:2648:do_file_write_iter()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0], ppos: 8388608, count: 614400 00000080:00200000:0.0:1769074787.744336:0:63053:0:(file.c:1994:ll_file_io_generic()) f200b.sanity-flr: write ppos: 8388608, bytes: 614400 00000080:00200000:0.0:1769074787.744346:0:63053:0:(file.c:1759:ll_io_set_mirror()) f200b.sanity-flr: desiginated mirror: 3 00000020:00200000:0.0:1769074787.744353:0:63053:0:(cl_io.c:214:cl_io_rw_init()) header@0000000006fe93a2[0x4, 2, [0x200000405:0x1fb:0x0] hash] 00000020:00200000:0.0:1769074787.744355:0:63053:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [8388608, 9003008) 2048 0 00000080:00200000:0.0:1769074787.744358:0:63053:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 31 restore needed 0 00000080:00200000:0.0:1769074787.744370:0:63053:0:(file.c:2072:ll_file_io_generic()) Range lock [2048, 2197] 00020000:00200000:0.0:1769074787.744377:0:63053:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 2 chunk: [8388608, 9003008) 9003008, 614400 00020000:00200000:0.0:1769074787.744381:0:63053:0:(lov_io.c:860:lov_io_iter_init()) component[12] flags 0x11 00000080:00200000:0.0:1769074787.744397:0:63053:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [2048, 2197] 00000008:00010000:0.0:1769074787.744409:0:63053:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:0.0:1769074787.744412:0:63053:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df4b0, osc lock ffff9b0345a6c980, flags 0x40000 00000080:00200000:0.0:1769074787.744420:0:63053:0:(vvp_io.c:1312:vvp_io_write_start()) f200b.sanity-flr: write [8388608, 9003008) append 0 append lockpos 0 00000080:00200000:0.0:1769074787.744434:0:63053:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), size=614400 (max 603979776), offset=8388608=0x800000, pages 150 (max 147456), lockless, parallel 00000020:00200000:0.0:1769074787.744602:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000080:00200000:0.0:1769074787.744605:0:63053:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 614400 count 614400 file_offset 9003008 00000080:00200000:0.0:1769074787.744615:0:63053:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 17576 00000020:00010000:0.0:1769074787.744619:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0343c7e5d8 (ffff9b0353152cf8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074787.744621:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343c7e5d8 (ffff9b0353152cf8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.744624:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df4b0 (ffff9b03531526c8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074787.744626:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343c7e5d8 (ffff9b0353152cf8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074787.744628:0:63053:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df4b0 (ffff9b0353152cf8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074787.744634:0:63053:0:(vvp_io.c:518:vvp_io_advance()) advancing 614400 bytes 00000020:00200000:0.0:1769074787.744639:0:63053:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1769074787.745925:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9840 00000008:00100000:0.0:1769074787.749838:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a39f80 - object 0x2c0000401:708 offset 4194304<>4808704 00000100:00100000:0.0:1769074787.749938:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616939264:192.168.204.160@tcp:4:lfs.0 00000100:00100000:0.0:1769074787.772677:0:2406:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9b0342ad2800: committing for last_committed 17179874582 gen 1 00000008:00010000:0.0:1769074787.772728:0:2406:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b03515b9840 00000020:00200000:0.0:1769074787.772735:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1769074787.772738:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1769074787.772739:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1769074787.772741:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1769074787.772743:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1769074787.772744:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1769074787.772745:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1769074787.772747:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1769074787.772748:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1769074787.772749:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1769074787.772751:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1769074787.772752:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1769074787.772754:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1769074787.772756:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1769074787.772757:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1769074787.772759:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1769074787.772760:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1769074787.772762:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1769074787.772763:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1769074787.772764:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1769074787.772766:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1769074787.772767:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1769074787.772769:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1769074787.772770:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1769074787.772772:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1769074787.772773:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1769074787.772775:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1769074787.772777:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1769074787.772778:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1769074787.772780:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1769074787.772781:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1769074787.772783:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1769074787.772785:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1769074787.772786:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1769074787.772788:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1769074787.772790:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1769074787.772792:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1769074787.772793:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1769074787.772795:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1769074787.772796:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1769074787.772798:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1769074787.772800:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1769074787.772801:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1769074787.772802:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1769074787.772804:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1769074787.772806:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1769074787.772807:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1769074787.772809:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1769074787.772811:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1769074787.772813:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1769074787.772815:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1769074787.772817:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1769074787.772818:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1769074787.772820:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1769074787.772822:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1769074787.772823:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1769074787.772825:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1769074787.772827:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1769074787.772829:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1769074787.772830:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1769074787.772832:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1769074787.772833:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1769074787.772835:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1769074787.772837:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1769074787.772839:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1769074787.772840:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1769074787.772920:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1769074787.772922:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1769074787.772924:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1769074787.772925:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1769074787.772926:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1769074787.772928:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1769074787.772929:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1769074787.772930:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1769074787.772931:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1769074787.772933:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1769074787.772934:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1769074787.772935:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1769074787.772937:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1769074787.772938:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1769074787.772940:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1769074787.772941:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1769074787.772942:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1769074787.772944:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1769074787.772945:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1769074787.772947:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1769074787.772948:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1769074787.772950:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1769074787.772951:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1769074787.772953:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1769074787.772955:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1769074787.772956:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1769074787.772958:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1769074787.772960:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1769074787.772961:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1769074787.772963:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1769074787.772965:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1769074787.772967:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1769074787.772968:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1769074787.772970:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1769074787.772971:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1769074787.772973:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1769074787.772975:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1769074787.772976:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1769074787.772978:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1769074787.772979:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1769074787.772981:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1769074787.772983:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1769074787.772984:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1769074787.772986:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1769074787.772987:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1769074787.772989:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1769074787.772990:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1769074787.772992:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1769074787.773082:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1769074787.773084:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1769074787.773086:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1769074787.773088:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1769074787.773090:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1769074787.773091:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1769074787.773093:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1769074787.773094:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1769074787.773096:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1769074787.773097:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1769074787.773099:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1769074787.773101:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1769074787.773102:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1769074787.773104:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1769074787.773106:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1769074787.773108:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1769074787.773109:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1769074787.773111:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1769074787.773112:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1769074787.773114:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1769074787.773115:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1769074787.773117:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1769074787.773119:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1769074787.773120:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1769074787.773122:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1769074787.773124:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1769074787.773125:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1769074787.773127:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1769074787.773129:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1769074787.773130:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1769074787.773132:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1769074787.773134:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1769074787.773136:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1769074787.773137:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1769074787.773139:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1769074787.773140:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1769074787.773196:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1769074787.773220:0:2406:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b03515b9840 00000100:00100000:0.0:1769074787.774057:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616939264:192.168.204.160@tcp:4:lfs.0 00000100:00100000:0.0:1769074787.926863:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074787.933566:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074787.941298:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625186816 00000100:00100000:0.0:1769074787.941309:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625186816 00000100:00100000:0.0:1769074787.941368:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8872 00000100:00100000:0.0:1769074787.941378:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8398:x1855007625186816:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074787.941396:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0341219e00/0xb7d6bf619879c22e lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd544b1 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.941471:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074787.941478:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8398:x1855007625186816:12345-192.168.204.160@tcp:104: Request processed in 102us (460us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074787.941485:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8872 00000100:00100000:0.0:1769074787.948975:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074787.960047:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074787.969802:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074787.978907:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625187328 00000100:00100000:0.0:1769074787.978917:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625187328 00000100:00100000:0.0:1769074787.978975:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8876 00000100:00100000:0.0:1769074787.978986:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8398:x1855007625187328:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074787.979156:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0341219c00/0xb7d6bf619879c23c lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd544bf expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.979286:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074787.979293:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8398:x1855007625187328:12345-192.168.204.160@tcp:104: Request processed in 309us (496us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074787.979299:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8876 00000100:00100000:0.0:1769074787.989644:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074787.989962:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625187456 00000100:00100000:0.0:1769074787.989974:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625187456 00000100:00100000:0.0:1769074787.990150:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8877 00000100:00100000:0.0:1769074787.990162:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a39180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6498:x1855007625187456:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074787.990193:0:55997:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.990209:0:55997:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.990218:0:55997:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 5/0,1 mode: --/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10000000020000 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.990450:0:55997:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10080000020000 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.990463:0:55997:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10080000020000 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.990474:0:55997:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 6/0,1 mode: PW/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x80000020000 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074787.990538:0:55997:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b03466c5e00) 00000100:00100000:0.0:1769074787.990594:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a39180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6498:x1855007625187456:12345-192.168.204.160@tcp:105: Request processed in 433us (960us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074787.990614:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8877 00000100:00100000:0.0:1769074787.998920:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.005522:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.010850:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.010895:0:55997:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074788.010914:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625187840 00000100:00100000:0.0:1769074788.010924:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625187840 00000100:00100000:0.0:1769074788.010984:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8880 00000100:00100000:0.0:1769074788.011641:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0344cc6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8398:x1855007625187840:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074788.011672:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c24a lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd544b8 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.012540:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074788.012554:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0344cc6050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8398:x1855007625187840:12345-192.168.204.160@tcp:104: Request processed in 1557us (1719us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074788.012566:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8880 00000100:00100000:0.0:1769074788.023774:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.037230:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000002:00010000:0.0:1769074788.102579:0:63056:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074788.102595:0:63056:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf6198798ec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ed2a expref: -99 pid: 61877 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074788.102606:0:63056:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf6198798ec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ed2a expref: -99 pid: 61877 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074788.102614:0:63056:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074788.102621:0:63056:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074788.102624:0:63056:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf6198798ec6 00000080:00010000:0.0:1769074788.102629:0:63056:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf6198798ec6 from it ffffb43642ef7bb0 00010000:00010000:0.0:1769074788.102632:0:63056:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf6198798ec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd4ed2a expref: -99 pid: 61877 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074788.102640:0:63056:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf6198798ec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ed2a expref: -99 pid: 61877 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074788.102653:0:63056:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074788.102745:0:63056:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 304000 00000080:00200080:0.0:1769074788.102782:0:63056:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=c0086915 arg=e1b3d0 00000100:00100000:0.0:1769074788.102852:0:63056:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:63056:1855007616945536:192.168.204.160@tcp:34:lfs.0 00000100:00100000:0.0:1769074788.105480:0:63056:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8bd80 going to sleep for 11 seconds 00000080:00200000:0.0:1769074788.157627:0:63062:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074788.157659:0:63062:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074788.157677:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf6198798ec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ed2a expref: -99 pid: 61877 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074788.157694:0:63062:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf6198798ec6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ed2a expref: -99 pid: 61877 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074788.157703:0:63062:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074788.157711:0:63062:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074788.157715:0:63062:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf6198798ec6 00000080:00010000:0.0:1769074788.157723:0:63062:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf6198798ec6 from it ffffb43642f87bf8 00010000:00010000:0.0:1769074788.157728:0:63062:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf6198798ec6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd4ed2a expref: -99 pid: 61877 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074788.157736:0:63062:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf6198798ec6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd4ed2a expref: -99 pid: 61877 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074788.157753:0:63062:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074788.157769:0:63062:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f200b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), file ffff9b0344c93680, open_flags c000, mode 0 opened 0 00000080:00200000:0.0:1769074788.157776:0:63062:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f200b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=open 00000002:00010000:0.0:1769074788.157791:0:63062:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f200b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0140001 00010000:00010000:0.0:1769074788.157863:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 63062 initiator: MDT0 00010000:00010000:0.0:1769074788.157877:0:63062:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63062 initiator: MDT0 00000020:00100000:0.0:1769074788.157889:0:63062:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074788.157901:0:63062:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3bb80 x1855007616945792/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074788.157912:0:63062:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 63062 initiator: MDT0 00000100:00100000:0.0:1769074788.157934:0:63062:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:63062:1855007616945792:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074788.158326:0:63062:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0344c8b480 going to sleep for 11 seconds 00000100:00100000:0.0:1769074788.214620:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.214754:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625190400 00000100:00100000:0.0:1769074788.214762:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625190400 00000100:00100000:0.0:1769074788.214808:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8883 00000100:00100000:0.0:1769074788.214817:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a39c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28448:x1855007625190400:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074788.214832:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c2a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000020000 nid: local remote: 0x7fe2323adfd545a6 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.214892:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074788.214898:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a39c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28448:x1855007625190400:12345-192.168.204.160@tcp:104: Request processed in 83us (280us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074788.214905:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8883 00010000:00010000:0.0:1769074788.214927:0:46917:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074788.214933:0:46917:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c2a5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000020000 nid: local remote: 0x7fe2323adfd545a6 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.214946:0:46917:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0350988800 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:0.0:1769074788.214952:0:46917:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c2a5 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800428400020000 nid: local remote: 0x7fe2323adfd545a6 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074788.214972:0:46917:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074788.214987:0:46917:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1769074788.215018:0:46917:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c2a5 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd545a6 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.215025:0:46917:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074788.215043:0:46917:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c2a5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd545a6 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.215050:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350988800/0xb7d6bf619879c2a5 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd545a6 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.215071:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074788.215073:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074788.215074:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074788.215081:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074788.215092:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616946560:192.168.204.160@tcp:103:ldlm_bl.0 00000080:00200000:0.0:1769074788.217265:0:63062:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [1280, 1535] 00000008:00010000:0.0:1769074788.217304:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1769074788.217311:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df4b0, osc lock ffff9b0345a6c980, flags 0x0 00000080:00200000:0.0:1769074788.217321:0:63062:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [5242880, 6291456) 00000080:00200000:0.0:1769074788.217340:0:63062:0:(vvp_io.c:886:vvp_io_read_start()) count:1048576 iocb pos:5242880 00000080:00200000:0.0:1769074788.217347:0:63062:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), size=1048576 (max 603979776), offset=5242880=0x500000, pages 256 (max 147456), lockless, parallel 00000020:00200000:0.0:1769074788.217588:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:0.0:1769074788.217592:0:63062:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 1048576 count 1048576 file_offset 6291456 00000020:00010000:0.0:1769074788.217599:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_release():245 00000020:00010000:0.0:1769074788.217603:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074788.217606:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df4b0 (ffff9b0353152b48/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074788.217609:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074788.217612:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df4b0 (ffff9b0353152d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074788.217620:0:63062:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:0.0:1769074788.217626:0:63062:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 6 chunk: [6291456, 7340032) 67108864, 1048576 00020000:00200000:0.0:1769074788.217630:0:63062:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00000008:00010000:0.0:1769074788.217820:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9840 00000008:00100000:0.0:1769074788.217916:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a3b800 - object 0x280000401:700 offset 2097152<>3145728 00000100:00100000:0.0:1769074788.217968:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616946688:192.168.204.160@tcp:3:lfs.0 00000080:00200000:0.0:1769074788.218086:0:63062:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [1536, 1791] 00000008:00010000:0.0:1769074788.218096:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1769074788.218099:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df4b0, osc lock ffff9b0345a6c980, flags 0x0 00000080:00200000:0.0:1769074788.218104:0:63062:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [6291456, 7340032) 00000080:00200000:0.0:1769074788.218111:0:63062:0:(vvp_io.c:886:vvp_io_read_start()) count:1048576 iocb pos:6291456 00000080:00200000:0.0:1769074788.218114:0:63062:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), size=1048576 (max 603979776), offset=6291456=0x600000, pages 256 (max 147456), lockless, parallel 00000020:00200000:0.0:1769074788.218537:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:0.0:1769074788.218541:0:63062:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 1048576 count 1048576 file_offset 7340032 00000020:00010000:0.0:1769074788.218547:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_release():245 00000020:00010000:0.0:1769074788.218549:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074788.218552:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df4b0 (ffff9b0353152248/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074788.218555:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074788.218558:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df4b0 (ffff9b0353152d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074788.218563:0:63062:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:0.0:1769074788.218567:0:63062:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 7 chunk: [7340032, 8388608) 67108864, 1048576 00020000:00200000:0.0:1769074788.218571:0:63062:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00000008:00010000:0.0:1769074788.218589:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b96c0 00000008:00100000:0.0:1769074788.218649:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a3a300 - object 0x2c0000401:703 offset 3145728<>4194304 00000100:00100000:0.0:1769074788.219480:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616946816:192.168.204.160@tcp:3:lfs.0 00000080:00200000:0.0:1769074788.219568:0:63062:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [1792, 2047] 00000008:00010000:0.0:1769074788.219585:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1769074788.219588:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df4b0, osc lock ffff9b0345a6c980, flags 0x0 00000080:00200000:0.0:1769074788.219594:0:63062:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [7340032, 8388608) 00000080:00200000:0.0:1769074788.219605:0:63062:0:(vvp_io.c:886:vvp_io_read_start()) count:1048576 iocb pos:7340032 00000080:00200000:0.0:1769074788.219610:0:63062:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), size=1048576 (max 603979776), offset=7340032=0x700000, pages 256 (max 147456), lockless, parallel 00000020:00200000:0.0:1769074788.220109:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:0.0:1769074788.220114:0:63062:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 1048576 count 1048576 file_offset 8388608 00000020:00010000:0.0:1769074788.220120:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_release():245 00000020:00010000:0.0:1769074788.220123:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074788.220126:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df4b0 (ffff9b0353152b48/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074788.220128:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074788.220131:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df4b0 (ffff9b0353152d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074788.220138:0:63062:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00020000:00200000:0.0:1769074788.220143:0:63062:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 8 chunk: [8388608, 9437184) 67108864, 1048576 00020000:00200000:0.0:1769074788.220147:0:63062:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00000008:00010000:0.0:1769074788.220168:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b90c0 00000008:00100000:0.0:1769074788.220496:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a39880 - object 0x280000401:700 offset 3145728<>4194304 00000100:00100000:0.0:1769074788.220535:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616946944:192.168.204.160@tcp:3:lfs.0 00000080:00200000:0.0:1769074788.220611:0:63062:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [2048, 2303] 00000008:00010000:0.0:1769074788.220624:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1769074788.220627:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df4b0, osc lock ffff9b0345a6c980, flags 0x0 00000080:00200000:0.0:1769074788.220632:0:63062:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [8388608, 9437184) 00000001:00010000:0.0:1769074788.220640:0:63062:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fb:0x0] 00000008:00010000:0.0:1769074788.220646:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.220649:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902a30, osc lock ffff9b0345a6cf70, flags 0x400001002 00000008:00010000:0.0:1769074788.220889:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.220893:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902a70, osc lock ffff9b0345a6dc80, flags 0x400001002 00000008:00010000:0.0:1769074788.220899:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.220901:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902ab0, osc lock ffff9b0345a6d7c0, flags 0x400001002 00000008:00010000:0.0:1769074788.220904:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.220906:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902af0, osc lock ffff9b0345a6da20, flags 0x400001002 00000008:00010000:0.0:1769074788.220909:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.220911:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902b30, osc lock ffff9b0345a6c390, flags 0x400001002 00000008:00010000:0.0:1769074788.220914:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.220915:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902b70, osc lock ffff9b0345a6cab0, flags 0x400001002 00010000:00010000:0.0:1769074788.220962:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.220975:0:63062:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.221005:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4200/0xb7d6bf619879c2ac lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2ba:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545ad expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074788.221025:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616947072:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074788.221080:0:63062:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4200/0xb7d6bf619879c2ac lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545ad expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.221094:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:0.0:1769074788.221098:0:63062:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4200/0xb7d6bf619879c2ac lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd545ad expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.221107:0:63062:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4200/0xb7d6bf619879c2ac lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545ad expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.221131:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4400/0xb7d6bf619879c2b3 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2bb:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545b4 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.221140:0:63062:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4400/0xb7d6bf619879c2b3 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545b4 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.221147:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:0.0:1769074788.221150:0:63062:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4400/0xb7d6bf619879c2b3 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd545b4 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.221157:0:63062:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4400/0xb7d6bf619879c2b3 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545b4 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.221168:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.221175:0:63062:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 2097151) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.222410:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:0.0:1769074788.222417:0:63062:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.222427:0:63062:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.222457:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c2c1 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545c2 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.223157:0:63062:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 8388607) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c2c1 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545c2 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.223169:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:0.0:1769074788.223172:0:63062:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c2c1 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd545c2 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.223179:0:63062:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c2c1 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545c2 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.223208:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c2c8 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2bc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545c9 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.223218:0:63062:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 8388607) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c2c8 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545c9 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.223225:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:0.0:1769074788.223228:0:63062:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c2c8 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd545c9 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.223261:0:63062:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c2c8 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd545c9 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.223276:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1769074788.234822:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.235484:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.235511:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.235534:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.237857:0:55997:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074788.237871:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625190784 00000100:00100000:0.0:1769074788.237879:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625190784 00000100:00100000:0.0:1769074788.237916:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625190912 00000100:00100000:0.0:1769074788.237918:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625190912 00000100:00100000:0.0:1769074788.237934:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8886 00000100:00100000:0.0:1769074788.237942:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a39500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:28445:x1855007625190784:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074788.237955:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c2c8 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800420000020000 nid: local remote: 0x7fe2323adfd545c9 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.244297:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:0.0:1769074788.244315:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a39500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:28445:x1855007625190784:12345-192.168.204.160@tcp:104: Request processed in 6366us (9494us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074788.244327:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8886 00000100:00100000:0.0:1769074788.244404:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616946560:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074788.259376:0:2407:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.259401:0:2407:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.259420:0:2407:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.259428:0:2407:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.259436:0:2407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 5/1,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074788.259450:0:2407:0:(osc_lock.c:179:osc_lock_lvb_update()) ### no dlm lock: acquired size 1048576, setting rss=1048576; leaving kms=0, end=18446744073709551615 00000020:00200000:0.0:1769074788.259458:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:0.0:1769074788.259469:0:2407:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 4/1,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.259476:0:2407:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 3/0,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x10206400000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.259484:0:2407:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 4/0,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x206400000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.259491:0:2407:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b03466c3600 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:0.0:1769074788.259496:0:2407:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 5/0,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x20e400000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.259507:0:2407:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074788.259514:0:2407:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 3/0,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.259526:0:2407:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3600/0xb7d6bf619879c2e4 lrc: 0/0,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x4a0f400000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074788.259553:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616947072:192.168.204.160@tcp:101:lfs.0 00000080:00200000:0.0:1769074788.260590:0:63062:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 17576 00000020:00010000:0.0:1769074788.260692:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314f908 (ffff9b0353152d88/0) at cl_lock_release():245 00000020:00010000:0.0:1769074788.260698:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314f908 (ffff9b0353152d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074788.260705:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902a30 (ffff9b0353152638/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074788.260709:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902a70 (ffff9b0353152098/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074788.260715:0:63062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4200/0xb7d6bf619879c2ac lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd545ad expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074788.260734:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902ab0 (ffff9b0342ca8128/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074788.260738:0:63062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c4400/0xb7d6bf619879c2b3 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd545b4 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074788.260749:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902af0 (ffff9b0342ca8488/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074788.260753:0:63062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c5e00/0xb7d6bf619879c2ba lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd545bb expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074788.260766:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902b30 (ffff9b0353152248/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074788.260770:0:63062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c4e00/0xb7d6bf619879c2c1 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd545c2 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074788.260782:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343902b70 (ffff9b0353152b48/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074788.260787:0:63062:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0350988e00/0xb7d6bf619879c2c8 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x804c69400020020 nid: local remote: 0x7fe2323adfd545c9 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074788.260797:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314f908 (ffff9b0353152d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074788.260801:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902a30 (ffff9b0353152d88/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074788.260805:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902a70 (ffff9b0353152d88/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074788.260807:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902ab0 (ffff9b0353152d88/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074788.260809:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902af0 (ffff9b0353152d88/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074788.260812:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902b30 (ffff9b0353152d88/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074788.260814:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343902b70 (ffff9b0353152d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074788.260824:0:63062:0:(vvp_io.c:886:vvp_io_read_start()) count:1048576 iocb pos:8388608 00000080:00200000:0.0:1769074788.260832:0:63062:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), size=1048576 (max 603979776), offset=8388608=0x800000, pages 256 (max 147456), lockless, parallel 00000020:00200000:0.0:1769074788.261092:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000080:00200000:0.0:1769074788.261096:0:63062:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 611424 count 611424 file_offset 9000032 00000020:00010000:0.0:1769074788.261104:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_release():245 00000020:00010000:0.0:1769074788.261106:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074788.261110:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df4b0 (ffff9b0353152248/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074788.261113:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343c7ed58 (ffff9b0353152d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074788.261115:0:63062:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df4b0 (ffff9b0353152d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074788.261122:0:63062:0:(vvp_io.c:518:vvp_io_advance()) advancing 611424 bytes 00020000:00200000:0.0:1769074788.261128:0:63062:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 8 chunk: [9000032, 9437184) 67108864, 437152 00020000:00200000:0.0:1769074788.261132:0:63062:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00000008:00010000:0.0:1769074788.261161:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9780 00000008:00100000:0.0:1769074788.261230:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a38000 - object 0x2c0000401:703 offset 4194304<>4805728 00000100:00100000:0.0:1769074788.261297:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616947840:192.168.204.160@tcp:3:lfs.0 00000080:00200000:0.0:1769074788.261519:0:63062:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [2197, 2303] 00000008:00010000:0.0:1769074788.261534:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1769074788.261539:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df4b0, osc lock ffff9b0345a6c980, flags 0x0 00000080:00200000:0.0:1769074788.261547:0:63062:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [9000032, 9437184) 00000001:00010000:0.0:1769074788.272138:0:63062:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fb:0x0] 00010000:00010000:0.0:1769074788.274367:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.274387:0:63062:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074788.274454:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616948096:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074788.281798:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.281813:0:63062:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074788.281880:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616948224:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074788.282747:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616948352:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074788.282805:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616948608:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074788.282856:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074788.282865:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616948480:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074788.317420:0:2407:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317448:0:2407:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54678 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317467:0:2407:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54678 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317480:0:2407:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54678 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074788.317496:0:2407:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54678 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.317509:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00010000:00010000:0.0:1769074788.317514:0:2407:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54678 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317523:0:2407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54678 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317531:0:2407:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54678 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317539:0:2407:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54678 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074788.317559:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616948096:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074788.317592:0:2407:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [2097152->8388607] (req 2097152->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317606:0:2407:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54694 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317614:0:2407:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54694 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317621:0:2407:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54694 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074788.317630:0:2407:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 4805728, setting rss=4805728; kms=4805728, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54694 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.317639:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00010000:00010000:0.0:1769074788.317643:0:2407:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54694 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317649:0:2407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54694 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317656:0:2407:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54694 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.317664:0:2407:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54694 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074788.317675:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616948480:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074788.328403:0:2406:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.328430:0:2406:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5467f expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.328443:0:2406:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5467f expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.328455:0:2406:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5467f expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074788.328471:0:2406:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd5467f expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.328483:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00010000:00010000:0.0:1769074788.328488:0:2406:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5467f expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.328496:0:2406:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5467f expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.328504:0:2406:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5467f expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.328512:0:2406:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5467f expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074788.328535:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616948224:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074788.356122:0:2407:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.356168:0:2407:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54686 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.356187:0:2407:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54686 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.356202:0:2407:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54686 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074788.356220:0:2407:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54686 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.356270:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074788.356275:0:2407:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54686 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.356283:0:2407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54686 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.356290:0:2407:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54686 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.356303:0:2407:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54686 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074788.356562:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616948352:192.168.204.160@tcp:101:lfs.0 00000008:00010000:0.0:1769074788.364690:0:2406:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b03515b90c0 00000020:00200000:0.0:1769074788.364704:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1769074788.364709:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1769074788.364711:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1769074788.364713:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1769074788.364714:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1769074788.364716:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1769074788.364718:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1769074788.364721:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1769074788.364722:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1769074788.364723:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1769074788.364725:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1769074788.364726:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1769074788.364728:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1769074788.364729:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1769074788.364731:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1769074788.364732:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1769074788.364733:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1769074788.364735:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1769074788.364736:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1769074788.364737:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1769074788.364739:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1769074788.364740:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1769074788.364741:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1769074788.364743:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1769074788.364745:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1769074788.364747:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1769074788.364748:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1769074788.364750:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1769074788.364751:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1769074788.364753:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1769074788.364754:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1769074788.364755:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1769074788.364757:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1769074788.364758:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1769074788.364760:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1769074788.364761:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1769074788.364763:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1769074788.364764:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1769074788.364766:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1769074788.364768:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1769074788.364770:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1769074788.364772:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1769074788.364774:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1769074788.364776:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1769074788.364777:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1769074788.364780:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1769074788.364781:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1769074788.364782:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1769074788.364784:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1769074788.364785:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1769074788.364787:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1769074788.364789:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1769074788.364790:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1769074788.364792:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1769074788.364793:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1769074788.364795:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1769074788.364798:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1769074788.364800:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1769074788.364802:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1769074788.364803:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1769074788.364805:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1769074788.364807:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1769074788.364808:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1769074788.364810:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1769074788.364811:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1769074788.364813:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1769074788.364814:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1769074788.364816:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1769074788.364817:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1769074788.364819:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1769074788.364820:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1769074788.364822:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1769074788.364823:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1769074788.364825:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1769074788.364827:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1769074788.364829:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1769074788.364831:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1769074788.364846:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1769074788.364848:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1769074788.364849:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1769074788.364851:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1769074788.364853:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1769074788.364855:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1769074788.364857:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1769074788.364859:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1769074788.364861:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1769074788.364862:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1769074788.364864:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1769074788.364866:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1769074788.364868:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1769074788.364870:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1769074788.364872:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1769074788.364874:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1769074788.364875:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1769074788.364876:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1769074788.364878:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1769074788.364880:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1769074788.364881:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1769074788.364883:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1769074788.364884:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1769074788.364886:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1769074788.364888:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1769074788.364889:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1769074788.364891:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1769074788.364892:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1769074788.364894:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1769074788.364896:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1769074788.364897:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1769074788.364899:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1769074788.364900:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1769074788.364902:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1769074788.364903:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1769074788.364905:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1769074788.364906:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1769074788.364908:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1769074788.364909:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1769074788.364911:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1769074788.364913:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1769074788.364914:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1769074788.364916:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1769074788.364918:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1769074788.364920:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1769074788.364922:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1769074788.364924:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1769074788.364926:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1769074788.364927:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1769074788.364929:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1769074788.364930:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1769074788.364931:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1769074788.364933:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1769074788.364935:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1769074788.364936:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1769074788.364937:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1769074788.364939:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1769074788.364940:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1769074788.364942:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1769074788.364944:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1769074788.364945:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1769074788.364947:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1769074788.364949:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1769074788.364951:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1769074788.364953:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1769074788.364955:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1769074788.364956:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1769074788.364958:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1769074788.364960:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1769074788.364961:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1769074788.364963:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1769074788.364964:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1769074788.364966:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1769074788.364968:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1769074788.364970:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1769074788.364972:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1769074788.364974:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1769074788.364976:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1769074788.364978:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1769074788.364979:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1769074788.364981:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1769074788.364982:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1769074788.364984:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1769074788.364985:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1769074788.364987:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1769074788.364988:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1769074788.364990:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1769074788.364991:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1769074788.364993:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1769074788.364994:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1769074788.364995:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1769074788.364997:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1769074788.364998:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1769074788.365000:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1769074788.365002:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1769074788.365004:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1769074788.365005:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1769074788.365007:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1769074788.365008:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1769074788.365010:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1769074788.365011:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1769074788.365013:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1769074788.365014:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1769074788.365016:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1769074788.365017:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1769074788.365019:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1769074788.365021:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1769074788.365022:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1769074788.365024:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1769074788.365025:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1769074788.365027:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1769074788.365028:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1769074788.365030:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1769074788.365031:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1769074788.365033:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1769074788.365034:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1769074788.365036:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1769074788.365038:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1769074788.365039:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1769074788.365041:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1769074788.365042:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1769074788.365044:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1769074788.365045:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1769074788.365047:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1769074788.365048:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1769074788.365050:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1769074788.365051:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1769074788.365053:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1769074788.365054:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1769074788.365056:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1769074788.365057:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1769074788.365059:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1769074788.365061:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1769074788.365063:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1769074788.365065:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1769074788.365067:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1769074788.365068:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1769074788.365070:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1769074788.365072:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1769074788.365074:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1769074788.365075:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1769074788.365077:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1769074788.365079:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1769074788.365081:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1769074788.365082:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1769074788.365084:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1769074788.365086:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1769074788.365088:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1769074788.365090:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1769074788.365091:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1769074788.365093:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1769074788.365095:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1769074788.365097:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1769074788.365098:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1769074788.365100:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1769074788.365101:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1769074788.365103:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1769074788.365104:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1769074788.365106:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1769074788.365107:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1769074788.365109:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1769074788.365111:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1769074788.365113:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1769074788.365114:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1769074788.365116:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1769074788.365118:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1769074788.365120:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1769074788.365122:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1769074788.365124:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1769074788.365126:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1769074788.365128:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1769074788.365130:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1769074788.365132:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1769074788.365133:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1769074788.365136:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1769074788.365137:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1769074788.365139:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1769074788.365141:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1769074788.365143:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1769074788.365267:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000008:00010000:0.0:1769074788.365273:0:2406:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b03515b90c0 00000100:00100000:0.0:1769074788.365299:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616946944:192.168.204.160@tcp:3:lfs.0 00000008:00010000:0.0:1769074788.365827:0:2406:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b03515b9840 00000020:00200000:0.0:1769074788.365830:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:0.0:1769074788.365842:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:0.0:1769074788.365844:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:0.0:1769074788.365846:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:0.0:1769074788.365848:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:0.0:1769074788.365850:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:0.0:1769074788.365851:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:0.0:1769074788.365853:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:0.0:1769074788.365855:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:0.0:1769074788.365856:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:0.0:1769074788.365857:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:0.0:1769074788.365858:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:0.0:1769074788.365860:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:0.0:1769074788.365861:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:0.0:1769074788.365863:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:0.0:1769074788.365865:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:0.0:1769074788.365867:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:0.0:1769074788.365868:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:0.0:1769074788.365870:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:0.0:1769074788.365872:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:0.0:1769074788.365874:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:0.0:1769074788.365876:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:0.0:1769074788.365877:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:0.0:1769074788.365879:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:0.0:1769074788.365881:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:0.0:1769074788.365883:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:0.0:1769074788.365884:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:0.0:1769074788.365885:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:0.0:1769074788.365886:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:0.0:1769074788.365888:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:0.0:1769074788.365889:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:0.0:1769074788.365890:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:0.0:1769074788.365891:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:0.0:1769074788.365893:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:0.0:1769074788.365895:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:0.0:1769074788.365897:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:0.0:1769074788.365898:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:0.0:1769074788.365900:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:0.0:1769074788.365902:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:0.0:1769074788.365903:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:0.0:1769074788.365905:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:0.0:1769074788.365906:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:0.0:1769074788.365908:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:0.0:1769074788.365910:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:0.0:1769074788.365911:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:0.0:1769074788.365913:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:0.0:1769074788.365915:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:0.0:1769074788.365916:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:0.0:1769074788.365918:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:0.0:1769074788.365919:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:0.0:1769074788.365921:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:0.0:1769074788.365923:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:0.0:1769074788.365925:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:0.0:1769074788.365927:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:0.0:1769074788.365928:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:0.0:1769074788.365930:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:0.0:1769074788.365931:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:0.0:1769074788.365933:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:0.0:1769074788.365935:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:0.0:1769074788.365936:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:0.0:1769074788.365937:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:0.0:1769074788.365939:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:0.0:1769074788.365940:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:0.0:1769074788.365942:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:0.0:1769074788.365944:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:0.0:1769074788.365945:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:0.0:1769074788.365946:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:0.0:1769074788.365947:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:0.0:1769074788.365948:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:0.0:1769074788.365950:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:0.0:1769074788.365951:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:0.0:1769074788.365952:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:0.0:1769074788.365954:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:0.0:1769074788.365955:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:0.0:1769074788.365957:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:0.0:1769074788.365958:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:0.0:1769074788.365960:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:0.0:1769074788.365961:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:0.0:1769074788.365963:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:0.0:1769074788.365964:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:0.0:1769074788.365966:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:0.0:1769074788.365967:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:0.0:1769074788.365969:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:0.0:1769074788.365970:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:0.0:1769074788.365972:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:0.0:1769074788.365974:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:0.0:1769074788.365975:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:0.0:1769074788.365976:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:0.0:1769074788.365978:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:0.0:1769074788.365979:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:0.0:1769074788.365980:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:0.0:1769074788.365982:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:0.0:1769074788.365983:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:0.0:1769074788.365985:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:0.0:1769074788.365987:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:0.0:1769074788.365988:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:0.0:1769074788.365990:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:0.0:1769074788.365992:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:0.0:1769074788.365993:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:0.0:1769074788.365995:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:0.0:1769074788.365997:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:0.0:1769074788.365998:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:0.0:1769074788.366000:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:0.0:1769074788.366001:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:0.0:1769074788.366003:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:0.0:1769074788.366005:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:0.0:1769074788.366006:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1769074788.366008:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1769074788.366009:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1769074788.366011:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1769074788.366013:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1769074788.366014:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1769074788.366015:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1769074788.366017:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1769074788.366020:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1769074788.366022:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1769074788.366023:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1769074788.366025:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1769074788.366027:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1769074788.366029:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1769074788.366031:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1769074788.366032:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1769074788.366034:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1769074788.366035:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1769074788.366037:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1769074788.366039:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1769074788.366041:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1769074788.366042:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1769074788.366044:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1769074788.366046:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1769074788.366047:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1769074788.366049:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1769074788.366051:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1769074788.366052:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1769074788.366054:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1769074788.366055:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1769074788.366057:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1769074788.366059:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1769074788.366060:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1769074788.366062:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1769074788.366064:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1769074788.366065:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1769074788.366067:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1769074788.366068:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1769074788.366069:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1769074788.366071:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1769074788.366073:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1769074788.366074:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1769074788.366076:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1769074788.366078:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1769074788.366079:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1769074788.366080:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1769074788.366082:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1769074788.366083:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1769074788.366085:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1769074788.366086:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1769074788.366089:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1769074788.366091:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1769074788.366092:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1769074788.366094:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1769074788.366095:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1769074788.366096:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1769074788.366098:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1769074788.366100:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1769074788.366101:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1769074788.366102:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1769074788.366104:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1769074788.366105:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1769074788.366106:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1769074788.366108:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1769074788.366109:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1769074788.366110:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1769074788.366111:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1769074788.366113:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1769074788.366114:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1769074788.366115:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1769074788.366117:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1769074788.366118:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1769074788.366121:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1769074788.366122:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1769074788.366124:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1769074788.366125:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1769074788.366127:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1769074788.366129:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1769074788.366130:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1769074788.366132:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1769074788.366133:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1769074788.366135:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1769074788.366137:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1769074788.366138:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1769074788.366140:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1769074788.366141:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1769074788.366143:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1769074788.366145:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1769074788.366146:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1769074788.366148:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1769074788.366150:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1769074788.366151:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1769074788.366153:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1769074788.366154:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1769074788.366156:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1769074788.366157:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1769074788.366159:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1769074788.366160:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1769074788.366162:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1769074788.366164:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1769074788.366165:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1769074788.366167:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1769074788.366168:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1769074788.366170:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1769074788.366172:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1769074788.366173:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1769074788.366175:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1769074788.366176:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1769074788.366178:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1769074788.366179:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1769074788.366181:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1769074788.366182:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1769074788.366184:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1769074788.366186:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1769074788.366188:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1769074788.366189:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1769074788.366191:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1769074788.366192:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1769074788.366194:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1769074788.366196:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1769074788.366198:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1769074788.366199:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1769074788.366201:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1769074788.366202:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1769074788.366204:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1769074788.366206:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1769074788.366207:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1769074788.366209:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1769074788.366210:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1769074788.366212:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1769074788.366214:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1769074788.366215:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1769074788.366217:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1769074788.366219:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1769074788.366220:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1769074788.366222:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1769074788.366224:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1769074788.366226:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1769074788.366227:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1769074788.366229:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1769074788.366231:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1769074788.366253:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1769074788.366255:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1769074788.366256:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1769074788.366258:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1769074788.366259:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1769074788.366261:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1769074788.366262:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1769074788.366264:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1769074788.366266:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1769074788.366351:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000008:00010000:0.0:1769074788.366356:0:2406:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b03515b9840 00000100:00100000:0.0:1769074788.366371:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616946688:192.168.204.160@tcp:3:lfs.0 00000008:00010000:0.0:1769074788.374175:0:2406:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b03515b9780 00000020:00200000:0.0:1769074788.374301:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1769074788.374303:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1769074788.374305:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1769074788.374307:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1769074788.374309:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1769074788.374311:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1769074788.374312:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1769074788.374314:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1769074788.374316:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1769074788.374317:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1769074788.374319:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1769074788.374320:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1769074788.374322:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1769074788.374324:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1769074788.374326:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1769074788.374327:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1769074788.374329:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1769074788.374330:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1769074788.374332:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1769074788.374333:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1769074788.374334:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1769074788.374336:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1769074788.374337:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1769074788.374339:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1769074788.374341:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1769074788.374342:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1769074788.374344:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1769074788.374346:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1769074788.374347:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1769074788.374349:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1769074788.374351:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1769074788.374353:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1769074788.374355:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1769074788.374356:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1769074788.374358:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1769074788.374359:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1769074788.374361:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1769074788.374363:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1769074788.374364:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1769074788.374366:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1769074788.374367:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1769074788.374369:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1769074788.374370:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1769074788.374371:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1769074788.374373:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1769074788.374375:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1769074788.374377:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1769074788.374378:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1769074788.374380:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1769074788.374382:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1769074788.374383:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1769074788.374385:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1769074788.374386:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1769074788.374388:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1769074788.374389:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1769074788.374391:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1769074788.374392:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1769074788.374394:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1769074788.374396:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1769074788.374397:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1769074788.374399:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1769074788.374402:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1769074788.374403:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1769074788.374404:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1769074788.374406:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1769074788.374408:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1769074788.374409:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1769074788.374410:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1769074788.374412:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1769074788.374414:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1769074788.374415:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1769074788.374417:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1769074788.374419:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1769074788.374420:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1769074788.374422:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1769074788.374424:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1769074788.374425:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1769074788.374427:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1769074788.374428:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1769074788.374430:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1769074788.374431:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1769074788.374433:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1769074788.374434:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1769074788.374435:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1769074788.374437:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1769074788.374439:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1769074788.374441:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1769074788.374442:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1769074788.374444:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1769074788.374445:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1769074788.374447:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1769074788.374448:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1769074788.374450:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1769074788.374452:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1769074788.374453:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1769074788.374455:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1769074788.374457:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1769074788.374458:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1769074788.374460:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1769074788.374461:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1769074788.374462:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1769074788.374464:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1769074788.374465:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1769074788.374467:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1769074788.374469:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1769074788.374471:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1769074788.374473:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1769074788.374474:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1769074788.374476:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1769074788.374478:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1769074788.374480:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1769074788.374481:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1769074788.374483:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1769074788.374484:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1769074788.374486:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1769074788.374488:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1769074788.374489:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1769074788.374491:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1769074788.374492:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1769074788.374493:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1769074788.374495:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1769074788.374497:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1769074788.374498:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1769074788.374500:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1769074788.374502:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1769074788.374504:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1769074788.374505:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1769074788.374506:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1769074788.374508:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1769074788.374509:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1769074788.374511:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1769074788.374512:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1769074788.374513:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1769074788.374515:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1769074788.374516:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1769074788.374518:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1769074788.374519:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1769074788.374521:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1769074788.374522:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1769074788.374524:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1769074788.374526:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1769074788.374528:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1769074788.374530:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1769074788.374531:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1769074788.374533:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1769074788.374534:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1769074788.374535:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1769074788.374537:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1769074788.374538:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1769074788.374540:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1769074788.374829:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1769074788.374890:0:2406:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b03515b9780 00000100:00100000:0.0:1769074788.374916:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616947840:192.168.204.160@tcp:3:lfs.0 00000080:00200080:0.0:1769074788.606881:0:63062:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8) cmd=400866b1 arg=1 00000080:00200008:0.0:1769074788.606905:0:63062:0:(file.c:6252:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074788.606916:0:63062:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8),name=f200b.sanity-flr 00000002:00010000:0.0:1769074788.606951:0:63062:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x1fb:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074788.606971:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074788.606987:0:63062:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074788.606996:0:63062:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074788.607005:0:63062:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x1fb:0x0](ffff9b0355ce63c8) 00000080:00010000:0.0:1769074788.607020:0:63062:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x1fb:0x0](ffff9b0355ce63c8) for lock 0xb7d6bf619879c2dd 00000080:00010000:0.0:1769074788.607028:0:63062:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c2dd from it ffffb43642f87cd8 00010000:00010000:0.0:1769074788.607032:0:63062:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074788.607047:0:63062:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00000080:00200008:0.0:1769074788.607075:0:63062:0:(file.c:2474:do_file_read_iter()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0], ppos: 9000032, count: 67108864 00000080:00200000:0.0:1769074788.607088:0:63062:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1769074788.607101:0:63062:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074788.607104:0:63062:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:0.0:1769074788.607120:0:63062:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 32 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074788.607127:0:63062:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:0.0:1769074788.607131:0:63062:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074788.607134:0:63062:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1769074788.607136:0:63062:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fb:0x0] 00000008:00010000:0.0:1769074788.607158:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.607162:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902a30, osc lock ffff9b0345a6cab0, flags 0x400000400001002 00000008:00010000:0.0:1769074788.607169:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.607170:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902a70, osc lock ffff9b0345a6c390, flags 0x400000400001002 00000008:00010000:0.0:1769074788.607417:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.607420:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902ab0, osc lock ffff9b0345a6da20, flags 0x400000400001002 00000008:00010000:0.0:1769074788.607459:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.607461:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902af0, osc lock ffff9b0345a6d7c0, flags 0x400000400001002 00000008:00010000:0.0:1769074788.607478:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.607480:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902b30, osc lock ffff9b0345a6dc80, flags 0x400000400001002 00000008:00010000:0.0:1769074788.607500:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.607502:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902b70, osc lock ffff9b0345a6cf70, flags 0x400000400001002 00000008:00010000:0.0:1769074788.607517:0:63062:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074788.607519:0:63062:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0343902bb0, osc lock ffff9b0345a6d300, flags 0x400000400001002 00010000:00010000:0.0:1769074788.607536:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c251 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5453d expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.607552:0:63062:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c251 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5453d expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074788.607566:0:63062:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c251 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd5453d expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.607577:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074788.607581:0:63062:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c251 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5453d expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.607590:0:63062:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c251 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5453d expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.607606:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0348214000/0xb7d6bf619879c258 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54544 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.607614:0:63062:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 3145727) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0348214000/0xb7d6bf619879c258 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54544 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074788.607623:0:63062:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 3145728, setting rss=3145728; kms=3145728, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0348214000/0xb7d6bf619879c258 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54544 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074788.607631:0:63062:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074788.607634:0:63062:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0348214000/0xb7d6bf619879c258 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54544 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.608099:0:63062:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0348214000/0xb7d6bf619879c258 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54544 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.608120:0:63062:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214800/0xb7d6bf619879c25f lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5454b expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.758530:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074788.765705:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00010000:00010000:0.0:1769074788.766663:0:46917:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074788.766671:0:46917:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c251 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd5453d expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.766680:0:46917:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365861400 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:0.0:1769074788.766687:0:46917:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c251 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800428400000000 nid: local remote: 0x7fe2323adfd5453d expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074788.766767:0:46917:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074788.766786:0:46917:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1769074788.766856:0:46917:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c251 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd5453d expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.766867:0:46917:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074788.766916:0:46917:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c251 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd5453d expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.766925:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c251 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd5453d expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.766953:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074788.766956:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074788.766958:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074788.766969:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074788.766988:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616950272:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074788.805176:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cebb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616950528:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074788.805272:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616950400:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074788.805315:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616950656:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074788.805350:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.805373:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625191808 00000100:00100000:0.0:1769074788.805379:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625191808 00000100:00100000:0.0:1769074788.805422:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8892 00000100:00100000:0.0:1769074788.805430:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a39880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:16437:x1855007625191808:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074788.805446:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0348215800/0xb7d6bf619879c266 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd54552 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.805568:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074788.805576:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a39880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:16437:x1855007625191808:12345-192.168.204.160@tcp:104: Request processed in 146us (225us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074788.805583:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8892 00000100:00100000:0.0:1769074788.805634:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074788.805650:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625191936 00000100:00100000:0.0:1769074788.805653:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625191936 00000100:00100000:0.0:1769074788.805669:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8893 00000100:00100000:0.0:1769074788.805675:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a38380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:28445:x1855007625191936:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074788.805683:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c274 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd54560 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074788.820328:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:0.0:1769074788.820347:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a38380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:28445:x1855007625191936:12345-192.168.204.160@tcp:104: Request processed in 14664us (14705us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074788.820360:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8893 00000080:00200008:0.0:1769074789.041078:0:63065:0:(file.c:424:ll_file_release()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), flags 140000 00010000:00010000:0.0:1769074789.041120:0:63065:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0359293800/0xb7d6bf619879c315 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x1fb:0x0].0x0 bits 0xd/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd546a2 expref: -99 pid: 63065 timeout: 0 lvb_type: 3 00000080:00200080:0.0:1769074789.129125:0:63068:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8) cmd=400866b1 arg=3 00000080:00200008:0.0:1769074789.129148:0:63068:0:(file.c:6252:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074789.129158:0:63068:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8),name=f200b.sanity-flr 00000002:00010000:0.0:1769074789.129312:0:63068:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x1fb:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074789.129332:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074789.129347:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074789.129354:0:63068:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074789.129362:0:63068:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x1fb:0x0](ffff9b0355ce63c8) 00000080:00010000:0.0:1769074789.129365:0:63068:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x1fb:0x0](ffff9b0355ce63c8) for lock 0xb7d6bf619879c2dd 00000080:00010000:0.0:1769074789.129372:0:63068:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c2dd from it ffffb43642fafcd8 00010000:00010000:0.0:1769074789.129377:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074789.129386:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00000080:00200008:0.0:1769074789.129422:0:63068:0:(file.c:2474:do_file_read_iter()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0], ppos: 0, count: 67108864 00000080:00200000:0.0:1769074789.129434:0:63068:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:0.0:1769074789.129448:0:63068:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074789.129451:0:63068:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:0.0:1769074789.129469:0:63068:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 32 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074789.129475:0:63068:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:0.0:1769074789.129479:0:63068:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074789.129481:0:63068:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1769074789.129484:0:63068:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fb:0x0] 00000008:00010000:0.0:1769074789.129538:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.129544:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0353015230, osc lock ffff9b0345a6d690, flags 0x400000400001002 00000008:00010000:0.0:1769074789.129575:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.129579:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0353015270, osc lock ffff9b0345a6c720, flags 0x400000400001002 00000008:00010000:0.0:1769074789.129602:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.129606:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03530152b0, osc lock ffff9b0345a6d0a0, flags 0x400000400001002 00000008:00010000:0.0:1769074789.129626:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.129628:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03530152f0, osc lock ffff9b0345a6c4c0, flags 0x400000400001002 00000008:00010000:0.0:1769074789.129766:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.129769:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0353015330, osc lock ffff9b0345a6d430, flags 0x400000400001002 00000008:00010000:0.0:1769074789.129786:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.129787:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0353015370, osc lock ffff9b0345a6c000, flags 0x400000400001002 00000008:00010000:0.0:1769074789.129802:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.129804:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03530153b0, osc lock ffff9b0345a6d8f0, flags 0x400000400001002 00010000:00010000:0.0:1769074789.129835:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1c00/0xb7d6bf619879c31c lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546e8 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.129857:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1c00/0xb7d6bf619879c31c lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546e8 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.129874:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.129880:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1c00/0xb7d6bf619879c31c lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd546e8 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.129890:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1c00/0xb7d6bf619879c31c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546e8 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.129917:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c323 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546ef expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.129928:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 3145727) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c323 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546ef expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.129937:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.129941:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c323 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd546ef expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.129950:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c323 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546ef expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.129969:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c32a lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546f6 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.129986:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 3145727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c32a lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546f6 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.129995:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.129999:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c32a lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd546f6 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130008:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c32a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546f6 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130030:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c331 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546fd expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130039:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c331 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546fd expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.130049:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.130052:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c331 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd546fd expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130061:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c331 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546fd expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130075:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd0a00/0xb7d6bf619879c338 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54704 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130090:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd0a00/0xb7d6bf619879c338 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54704 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.130098:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.130102:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd0a00/0xb7d6bf619879c338 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54704 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130110:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd0a00/0xb7d6bf619879c338 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54704 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130123:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c33f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5470b expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130132:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (4194304 16777215) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c33f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5470b expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.130140:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.130142:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c33f lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5470b expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130150:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c33f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5470b expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130160:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214600/0xb7d6bf619879c27b lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x2c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54567 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130379:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (4194304 16777215) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214600/0xb7d6bf619879c27b lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x2c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54567 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.130388:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.130391:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214600/0xb7d6bf619879c27b lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54567 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.130398:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214600/0xb7d6bf619879c27b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54567 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.130406:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074789.130419:0:63068:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 35136 00000020:00010000:0.0:1769074789.130424:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035312a6c8 (ffff9b0342b7d998/0) at cl_lock_release():245 00000020:00010000:0.0:1769074789.130427:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035312a6c8 (ffff9b0342b7d998/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.130431:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0353015230 (ffff9b0342b7d098/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.130435:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0353015270 (ffff9b0342b7dea8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.130437:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03530152b0 (ffff9b0342b7d008/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.130440:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03530152f0 (ffff9b0342b7d638/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.130443:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0353015330 (ffff9b0342b7dbd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.130445:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0353015370 (ffff9b0342b7d488/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.130448:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03530153b0 (ffff9b0342b7db48/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.130450:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035312a6c8 (ffff9b0342b7d998/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.130453:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0353015230 (ffff9b0342b7d998/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.130458:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0353015270 (ffff9b0342b7d998/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.130461:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03530152b0 (ffff9b0342b7d998/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.130463:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03530152f0 (ffff9b0342b7d998/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.130465:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0353015330 (ffff9b0342b7d998/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.130467:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0353015370 (ffff9b0342b7d998/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.130469:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03530153b0 (ffff9b0342b7d998/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074789.130490:0:63068:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 32 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074789.130496:0:63068:0:(file.c:2367:file_read_confine_iter()) f200b.sanity-flr: read [0, 67108864] over eof, kms 9000032, file_size 9000032. 00000080:00200000:0.0:1769074789.130503:0:63068:0:(file.c:1994:ll_file_io_generic()) f200b.sanity-flr: read ppos: 0, bytes: 9000032 00000080:00200000:0.0:1769074789.130512:0:63068:0:(file.c:1759:ll_io_set_mirror()) f200b.sanity-flr: desiginated mirror: 3 00000020:00200000:0.0:1769074789.130519:0:63068:0:(cl_io.c:214:cl_io_rw_init()) header@0000000006fe93a2[0x4, 2, [0x200000405:0x1fb:0x0] hash] 00000020:00200000:0.0:1769074789.130520:0:63068:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 9000032) 0 0 00000080:00200000:0.0:1769074789.130523:0:63068:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 32 restore needed 0 00000080:00200000:0.0:1769074789.130535:0:63068:0:(file.c:2072:ll_file_io_generic()) Range lock [0, 2197] 00020000:00200000:0.0:1769074789.130544:0:63068:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4194304) 9000032, 4194304 00020000:00200000:0.0:1769074789.130549:0:63068:0:(lov_io.c:860:lov_io_iter_init()) component[10] flags 0x10 00000080:00200000:0.0:1769074789.130565:0:63068:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 1023] 00000008:00010000:0.0:1769074789.130575:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1769074789.130578:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df0b0, osc lock ffff9b0345a6d8f0, flags 0x0 00000080:00200000:0.0:1769074789.130584:0:63068:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [0, 4194304) 00000080:00200000:0.0:1769074789.130592:0:63068:0:(vvp_io.c:886:vvp_io_read_start()) count:4194304 iocb pos:0 00000080:00200000:0.0:1769074789.130598:0:63068:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), size=4194304 (max 603979776), offset=0=0x0, pages 1024 (max 147456), lockless, parallel 00000020:00200000:0.0:1769074789.132360:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1024 items remaining. 00000080:00200000:0.0:1769074789.132370:0:63068:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 4194304 count 4194304 file_offset 4194304 00000020:00010000:0.0:1769074789.132380:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b034396ed58 (ffff9b0342b7dd88/0) at cl_lock_release():245 00000020:00010000:0.0:1769074789.132384:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b034396ed58 (ffff9b0342b7dd88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.132388:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df0b0 (ffff9b0342b7d998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.132391:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b034396ed58 (ffff9b0342b7dd88/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.132394:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df0b0 (ffff9b0342b7dd88/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074789.132404:0:63068:0:(vvp_io.c:518:vvp_io_advance()) advancing 4194304 bytes 00020000:00200000:0.0:1769074789.132411:0:63068:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 4194304 chunk: [4194304, 8388608) 9000032, 4194304 00020000:00200000:0.0:1769074789.132414:0:63068:0:(lov_io.c:860:lov_io_iter_init()) component[11] flags 0x10 00000008:00010000:0.0:1769074789.132505:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9300 00000008:00100000:0.0:1769074789.132729:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a38e00 - object 0x2c0000401:701 offset 0<>4194304 00000008:00010000:0.0:1769074789.135749:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b0347fb6c00 00000008:00100000:0.0:1769074789.139156:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a3a680 - object 0x2c0000401:707 offset 4194304<>8388608 00000008:00010000:0.0:1769074789.139325:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b0347fb6600 00000008:00100000:0.0:1769074789.139429:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a38a80 - object 0x2c0000401:708 offset 4194304<>4805728 00000100:00100000:0.0:1769074789.139472:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616952064:192.168.204.160@tcp:3:lfs.0 00000100:00100000:0.0:1769074789.139595:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074789.139607:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616951936:192.168.204.160@tcp:3:lfs.0 00000100:00100000:0.0:1769074789.151076:0:39330:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074789.151096:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625192320 00000100:00100000:0.0:1769074789.151107:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625192320 00000100:00100000:0.0:1769074789.151155:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8894 00000100:00100000:0.0:1769074789.151176:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0350845c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:16437:x1855007625192320:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074789.151194:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03453e8000/0xb7d6bf619879c289 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd54583 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.154463:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074789.154478:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0350845c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:16437:x1855007625192320:12345-192.168.204.160@tcp:104: Request processed in 3309us (3586us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074789.154487:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8894 00010000:00010000:0.0:1769074789.165816:0:46917:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074789.165834:0:46917:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c290 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd5458a expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.165843:0:46917:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b03453e9600 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:0.0:1769074789.165851:0:46917:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c290 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x428400000000 nid: local remote: 0x7fe2323adfd5458a expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.165897:0:46917:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c290 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd5458a expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.165904:0:46917:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074789.165925:0:46917:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c290 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd5458a expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.165933:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c290 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd5458a expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.165954:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074789.165956:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074789.165958:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074789.165971:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074789.165986:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616952448:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074789.221272:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616952448:192.168.204.160@tcp:103:ldlm_bl.0 00000008:00010000:0.0:1769074789.281864:0:2406:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b0347fb6600 00000020:00200000:0.0:1769074789.281877:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:0.0:1769074789.281881:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:0.0:1769074789.281883:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:0.0:1769074789.281885:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:0.0:1769074789.281887:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:0.0:1769074789.281888:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:0.0:1769074789.281890:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:0.0:1769074789.281892:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:0.0:1769074789.281893:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:0.0:1769074789.281895:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:0.0:1769074789.281897:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:0.0:1769074789.281898:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:0.0:1769074789.281900:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:0.0:1769074789.281901:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:0.0:1769074789.281903:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:0.0:1769074789.281904:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:0.0:1769074789.281906:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:0.0:1769074789.281907:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:0.0:1769074789.281909:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:0.0:1769074789.281910:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:0.0:1769074789.281912:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:0.0:1769074789.281913:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:0.0:1769074789.281915:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:0.0:1769074789.281917:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:0.0:1769074789.281918:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:0.0:1769074789.281920:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:0.0:1769074789.281922:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:0.0:1769074789.281923:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:0.0:1769074789.281925:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:0.0:1769074789.281927:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:0.0:1769074789.281928:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:0.0:1769074789.281930:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:0.0:1769074789.281931:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:0.0:1769074789.281933:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:0.0:1769074789.281935:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:0.0:1769074789.281936:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:0.0:1769074789.281938:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:0.0:1769074789.281939:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:0.0:1769074789.281941:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:0.0:1769074789.281942:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:0.0:1769074789.281944:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:0.0:1769074789.281946:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:0.0:1769074789.281948:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:0.0:1769074789.281950:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:0.0:1769074789.281952:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:0.0:1769074789.281953:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:0.0:1769074789.281955:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:0.0:1769074789.281957:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:0.0:1769074789.281958:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:0.0:1769074789.281959:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:0.0:1769074789.281961:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:0.0:1769074789.281963:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:0.0:1769074789.281964:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:0.0:1769074789.281966:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:0.0:1769074789.281967:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:0.0:1769074789.281969:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:0.0:1769074789.281970:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:0.0:1769074789.281972:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:0.0:1769074789.281973:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:0.0:1769074789.281975:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:0.0:1769074789.281976:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:0.0:1769074789.281977:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:0.0:1769074789.281979:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:0.0:1769074789.281980:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:0.0:1769074789.281982:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:0.0:1769074789.281983:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:0.0:1769074789.281985:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:0.0:1769074789.281987:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:0.0:1769074789.281988:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:0.0:1769074789.281989:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:0.0:1769074789.281991:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:0.0:1769074789.281992:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:0.0:1769074789.281993:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:0.0:1769074789.281995:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:0.0:1769074789.281996:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:0.0:1769074789.281998:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:0.0:1769074789.282000:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:0.0:1769074789.282001:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:0.0:1769074789.282003:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:0.0:1769074789.282004:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:0.0:1769074789.282006:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:0.0:1769074789.282008:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:0.0:1769074789.282009:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:0.0:1769074789.282011:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:0.0:1769074789.282012:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:0.0:1769074789.282014:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:0.0:1769074789.282015:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:0.0:1769074789.282017:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:0.0:1769074789.282019:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:0.0:1769074789.282020:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:0.0:1769074789.282022:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:0.0:1769074789.282024:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:0.0:1769074789.282026:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:0.0:1769074789.282027:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:0.0:1769074789.282029:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:0.0:1769074789.282031:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:0.0:1769074789.282032:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:0.0:1769074789.282034:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:0.0:1769074789.282036:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:0.0:1769074789.282037:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:0.0:1769074789.282039:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:0.0:1769074789.282040:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:0.0:1769074789.282042:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:0.0:1769074789.282044:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:0.0:1769074789.282045:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:0.0:1769074789.282047:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:0.0:1769074789.282049:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:0.0:1769074789.282050:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:0.0:1769074789.282052:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:0.0:1769074789.282053:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:0.0:1769074789.282055:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:0.0:1769074789.282056:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:0.0:1769074789.282058:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:0.0:1769074789.282059:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:0.0:1769074789.282061:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:0.0:1769074789.282062:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:0.0:1769074789.282064:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:0.0:1769074789.282066:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:0.0:1769074789.282067:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:0.0:1769074789.282069:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:0.0:1769074789.282070:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:0.0:1769074789.282071:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:0.0:1769074789.282073:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:0.0:1769074789.282074:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:0.0:1769074789.282076:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:0.0:1769074789.282077:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:0.0:1769074789.282078:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:0.0:1769074789.282080:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:0.0:1769074789.282081:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:0.0:1769074789.282082:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:0.0:1769074789.282084:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:0.0:1769074789.282085:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:0.0:1769074789.282086:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:0.0:1769074789.282088:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:0.0:1769074789.282089:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:0.0:1769074789.282090:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:0.0:1769074789.282091:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:0.0:1769074789.282093:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:0.0:1769074789.282094:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:0.0:1769074789.282096:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:0.0:1769074789.282097:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:0.0:1769074789.282099:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:0.0:1769074789.282100:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:0.0:1769074789.282102:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:0.0:1769074789.282103:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:0.0:1769074789.282105:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:0.0:1769074789.282106:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:0.0:1769074789.282108:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:0.0:1769074789.282110:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:0.0:1769074789.282111:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1769074789.282162:0:2406:0:(cl_io.c:1316:cl_sub_dio_end()) finishing unaligned dio read aio->cda_bytes 611424 00000020:00200000:0.0:1769074789.282172:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 611424, offset 0, pos 8388608, cdp_page_count 150 00000020:00200000:0.0:1769074789.282179:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 607328, offset 0, pos 8392704, cdp_page_count 150 00000020:00200000:0.0:1769074789.282182:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 603232, offset 0, pos 8396800, cdp_page_count 150 00000020:00200000:0.0:1769074789.282186:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 599136, offset 0, pos 8400896, cdp_page_count 150 00000020:00200000:0.0:1769074789.282189:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 595040, offset 0, pos 8404992, cdp_page_count 150 00000020:00200000:0.0:1769074789.282192:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 590944, offset 0, pos 8409088, cdp_page_count 150 00000020:00200000:0.0:1769074789.282196:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 586848, offset 0, pos 8413184, cdp_page_count 150 00000020:00200000:0.0:1769074789.282199:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 582752, offset 0, pos 8417280, cdp_page_count 150 00000020:00200000:0.0:1769074789.282201:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 578656, offset 0, pos 8421376, cdp_page_count 150 00000020:00200000:0.0:1769074789.282204:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 574560, offset 0, pos 8425472, cdp_page_count 150 00000020:00200000:0.0:1769074789.282208:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 570464, offset 0, pos 8429568, cdp_page_count 150 00000020:00200000:0.0:1769074789.282211:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 566368, offset 0, pos 8433664, cdp_page_count 150 00000020:00200000:0.0:1769074789.282213:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 562272, offset 0, pos 8437760, cdp_page_count 150 00000020:00200000:0.0:1769074789.282216:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 558176, offset 0, pos 8441856, cdp_page_count 150 00000020:00200000:0.0:1769074789.282219:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 554080, offset 0, pos 8445952, cdp_page_count 150 00000020:00200000:0.0:1769074789.282222:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 549984, offset 0, pos 8450048, cdp_page_count 150 00000020:00200000:0.0:1769074789.282225:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 545888, offset 0, pos 8454144, cdp_page_count 150 00000020:00200000:0.0:1769074789.282228:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 541792, offset 0, pos 8458240, cdp_page_count 150 00000020:00200000:0.0:1769074789.282263:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 537696, offset 0, pos 8462336, cdp_page_count 150 00000020:00200000:0.0:1769074789.282268:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 533600, offset 0, pos 8466432, cdp_page_count 150 00000020:00200000:0.0:1769074789.282271:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 529504, offset 0, pos 8470528, cdp_page_count 150 00000020:00200000:0.0:1769074789.282276:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 525408, offset 0, pos 8474624, cdp_page_count 150 00000020:00200000:0.0:1769074789.282279:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 521312, offset 0, pos 8478720, cdp_page_count 150 00000020:00200000:0.0:1769074789.282282:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 517216, offset 0, pos 8482816, cdp_page_count 150 00000020:00200000:0.0:1769074789.282285:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 513120, offset 0, pos 8486912, cdp_page_count 150 00000020:00200000:0.0:1769074789.282289:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 509024, offset 0, pos 8491008, cdp_page_count 150 00000020:00200000:0.0:1769074789.282291:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 504928, offset 0, pos 8495104, cdp_page_count 150 00000020:00200000:0.0:1769074789.282294:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 500832, offset 0, pos 8499200, cdp_page_count 150 00000020:00200000:0.0:1769074789.282297:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 496736, offset 0, pos 8503296, cdp_page_count 150 00000020:00200000:0.0:1769074789.282299:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 492640, offset 0, pos 8507392, cdp_page_count 150 00000020:00200000:0.0:1769074789.282303:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 488544, offset 0, pos 8511488, cdp_page_count 150 00000020:00200000:0.0:1769074789.282306:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 484448, offset 0, pos 8515584, cdp_page_count 150 00000020:00200000:0.0:1769074789.282308:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 480352, offset 0, pos 8519680, cdp_page_count 150 00000020:00200000:0.0:1769074789.282310:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 476256, offset 0, pos 8523776, cdp_page_count 150 00000020:00200000:0.0:1769074789.282314:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 472160, offset 0, pos 8527872, cdp_page_count 150 00000020:00200000:0.0:1769074789.282317:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 468064, offset 0, pos 8531968, cdp_page_count 150 00000020:00200000:0.0:1769074789.282319:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 463968, offset 0, pos 8536064, cdp_page_count 150 00000020:00200000:0.0:1769074789.282322:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 459872, offset 0, pos 8540160, cdp_page_count 150 00000020:00200000:0.0:1769074789.282325:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 455776, offset 0, pos 8544256, cdp_page_count 150 00000020:00200000:0.0:1769074789.282328:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 451680, offset 0, pos 8548352, cdp_page_count 150 00000020:00200000:0.0:1769074789.282330:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 447584, offset 0, pos 8552448, cdp_page_count 150 00000020:00200000:0.0:1769074789.282333:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 443488, offset 0, pos 8556544, cdp_page_count 150 00000020:00200000:0.0:1769074789.282336:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 439392, offset 0, pos 8560640, cdp_page_count 150 00000020:00200000:0.0:1769074789.282339:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 435296, offset 0, pos 8564736, cdp_page_count 150 00000020:00200000:0.0:1769074789.282342:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 431200, offset 0, pos 8568832, cdp_page_count 150 00000020:00200000:0.0:1769074789.282345:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 427104, offset 0, pos 8572928, cdp_page_count 150 00000020:00200000:0.0:1769074789.282348:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 423008, offset 0, pos 8577024, cdp_page_count 150 00000020:00200000:0.0:1769074789.282351:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 418912, offset 0, pos 8581120, cdp_page_count 150 00000020:00200000:0.0:1769074789.282354:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 414816, offset 0, pos 8585216, cdp_page_count 150 00000020:00200000:0.0:1769074789.282357:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 410720, offset 0, pos 8589312, cdp_page_count 150 00000020:00200000:0.0:1769074789.282360:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 406624, offset 0, pos 8593408, cdp_page_count 150 00000020:00200000:0.0:1769074789.282363:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 402528, offset 0, pos 8597504, cdp_page_count 150 00000020:00200000:0.0:1769074789.282366:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 398432, offset 0, pos 8601600, cdp_page_count 150 00000020:00200000:0.0:1769074789.282369:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 394336, offset 0, pos 8605696, cdp_page_count 150 00000020:00200000:0.0:1769074789.282372:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 390240, offset 0, pos 8609792, cdp_page_count 150 00000020:00200000:0.0:1769074789.282375:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 386144, offset 0, pos 8613888, cdp_page_count 150 00000020:00200000:0.0:1769074789.282379:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 382048, offset 0, pos 8617984, cdp_page_count 150 00000020:00200000:0.0:1769074789.282381:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 377952, offset 0, pos 8622080, cdp_page_count 150 00000020:00200000:0.0:1769074789.282384:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 373856, offset 0, pos 8626176, cdp_page_count 150 00000020:00200000:0.0:1769074789.282387:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 369760, offset 0, pos 8630272, cdp_page_count 150 00000020:00200000:0.0:1769074789.282390:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 365664, offset 0, pos 8634368, cdp_page_count 150 00000020:00200000:0.0:1769074789.282393:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 361568, offset 0, pos 8638464, cdp_page_count 150 00000020:00200000:0.0:1769074789.282396:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 357472, offset 0, pos 8642560, cdp_page_count 150 00000020:00200000:0.0:1769074789.282399:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 353376, offset 0, pos 8646656, cdp_page_count 150 00000020:00200000:0.0:1769074789.282402:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 349280, offset 0, pos 8650752, cdp_page_count 150 00000020:00200000:0.0:1769074789.282405:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 345184, offset 0, pos 8654848, cdp_page_count 150 00000020:00200000:0.0:1769074789.282408:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 341088, offset 0, pos 8658944, cdp_page_count 150 00000020:00200000:0.0:1769074789.282412:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 336992, offset 0, pos 8663040, cdp_page_count 150 00000020:00200000:0.0:1769074789.282416:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 332896, offset 0, pos 8667136, cdp_page_count 150 00000020:00200000:0.0:1769074789.282420:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 328800, offset 0, pos 8671232, cdp_page_count 150 00000020:00200000:0.0:1769074789.282424:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 324704, offset 0, pos 8675328, cdp_page_count 150 00000020:00200000:0.0:1769074789.282427:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 320608, offset 0, pos 8679424, cdp_page_count 150 00000020:00200000:0.0:1769074789.282431:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 316512, offset 0, pos 8683520, cdp_page_count 150 00000020:00200000:0.0:1769074789.282434:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 312416, offset 0, pos 8687616, cdp_page_count 150 00000020:00200000:0.0:1769074789.282438:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 308320, offset 0, pos 8691712, cdp_page_count 150 00000020:00200000:0.0:1769074789.282442:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 304224, offset 0, pos 8695808, cdp_page_count 150 00000020:00200000:0.0:1769074789.282446:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 300128, offset 0, pos 8699904, cdp_page_count 150 00000020:00200000:0.0:1769074789.282450:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 296032, offset 0, pos 8704000, cdp_page_count 150 00000020:00200000:0.0:1769074789.282453:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 291936, offset 0, pos 8708096, cdp_page_count 150 00000020:00200000:0.0:1769074789.282457:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 287840, offset 0, pos 8712192, cdp_page_count 150 00000020:00200000:0.0:1769074789.282461:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 283744, offset 0, pos 8716288, cdp_page_count 150 00000020:00200000:0.0:1769074789.282464:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 279648, offset 0, pos 8720384, cdp_page_count 150 00000020:00200000:0.0:1769074789.282467:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 275552, offset 0, pos 8724480, cdp_page_count 150 00000020:00200000:0.0:1769074789.282471:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 271456, offset 0, pos 8728576, cdp_page_count 150 00000020:00200000:0.0:1769074789.282474:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 267360, offset 0, pos 8732672, cdp_page_count 150 00000020:00200000:0.0:1769074789.282477:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 263264, offset 0, pos 8736768, cdp_page_count 150 00000020:00200000:0.0:1769074789.282480:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 259168, offset 0, pos 8740864, cdp_page_count 150 00000020:00200000:0.0:1769074789.282482:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 255072, offset 0, pos 8744960, cdp_page_count 150 00000020:00200000:0.0:1769074789.282485:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 250976, offset 0, pos 8749056, cdp_page_count 150 00000020:00200000:0.0:1769074789.282488:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 246880, offset 0, pos 8753152, cdp_page_count 150 00000020:00200000:0.0:1769074789.282490:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 242784, offset 0, pos 8757248, cdp_page_count 150 00000020:00200000:0.0:1769074789.282493:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 238688, offset 0, pos 8761344, cdp_page_count 150 00000020:00200000:0.0:1769074789.282496:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 234592, offset 0, pos 8765440, cdp_page_count 150 00000020:00200000:0.0:1769074789.282498:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 230496, offset 0, pos 8769536, cdp_page_count 150 00000020:00200000:0.0:1769074789.282502:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 226400, offset 0, pos 8773632, cdp_page_count 150 00000020:00200000:0.0:1769074789.282505:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 222304, offset 0, pos 8777728, cdp_page_count 150 00000020:00200000:0.0:1769074789.282508:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 218208, offset 0, pos 8781824, cdp_page_count 150 00000020:00200000:0.0:1769074789.282511:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 214112, offset 0, pos 8785920, cdp_page_count 150 00000020:00200000:0.0:1769074789.282515:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 210016, offset 0, pos 8790016, cdp_page_count 150 00000020:00200000:0.0:1769074789.282518:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 205920, offset 0, pos 8794112, cdp_page_count 150 00000020:00200000:0.0:1769074789.282521:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 201824, offset 0, pos 8798208, cdp_page_count 150 00000020:00200000:0.0:1769074789.282524:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 197728, offset 0, pos 8802304, cdp_page_count 150 00000020:00200000:0.0:1769074789.282526:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 193632, offset 0, pos 8806400, cdp_page_count 150 00000020:00200000:0.0:1769074789.282529:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 189536, offset 0, pos 8810496, cdp_page_count 150 00000020:00200000:0.0:1769074789.282532:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 185440, offset 0, pos 8814592, cdp_page_count 150 00000020:00200000:0.0:1769074789.282535:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 181344, offset 0, pos 8818688, cdp_page_count 150 00000020:00200000:0.0:1769074789.282538:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 177248, offset 0, pos 8822784, cdp_page_count 150 00000020:00200000:0.0:1769074789.282541:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 173152, offset 0, pos 8826880, cdp_page_count 150 00000020:00200000:0.0:1769074789.282543:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 169056, offset 0, pos 8830976, cdp_page_count 150 00000020:00200000:0.0:1769074789.282546:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 164960, offset 0, pos 8835072, cdp_page_count 150 00000020:00200000:0.0:1769074789.282549:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 160864, offset 0, pos 8839168, cdp_page_count 150 00000020:00200000:0.0:1769074789.282553:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 156768, offset 0, pos 8843264, cdp_page_count 150 00000020:00200000:0.0:1769074789.282556:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 152672, offset 0, pos 8847360, cdp_page_count 150 00000020:00200000:0.0:1769074789.282559:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 148576, offset 0, pos 8851456, cdp_page_count 150 00000020:00200000:0.0:1769074789.282561:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 144480, offset 0, pos 8855552, cdp_page_count 150 00000020:00200000:0.0:1769074789.282564:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 140384, offset 0, pos 8859648, cdp_page_count 150 00000020:00200000:0.0:1769074789.282567:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 136288, offset 0, pos 8863744, cdp_page_count 150 00000020:00200000:0.0:1769074789.282570:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 132192, offset 0, pos 8867840, cdp_page_count 150 00000020:00200000:0.0:1769074789.282573:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 128096, offset 0, pos 8871936, cdp_page_count 150 00000020:00200000:0.0:1769074789.282576:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 124000, offset 0, pos 8876032, cdp_page_count 150 00000020:00200000:0.0:1769074789.282579:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 119904, offset 0, pos 8880128, cdp_page_count 150 00000020:00200000:0.0:1769074789.282583:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 115808, offset 0, pos 8884224, cdp_page_count 150 00000020:00200000:0.0:1769074789.282586:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 111712, offset 0, pos 8888320, cdp_page_count 150 00000020:00200000:0.0:1769074789.282589:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 107616, offset 0, pos 8892416, cdp_page_count 150 00000020:00200000:0.0:1769074789.282592:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 103520, offset 0, pos 8896512, cdp_page_count 150 00000020:00200000:0.0:1769074789.282595:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 99424, offset 0, pos 8900608, cdp_page_count 150 00000020:00200000:0.0:1769074789.282598:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 95328, offset 0, pos 8904704, cdp_page_count 150 00000020:00200000:0.0:1769074789.282601:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 91232, offset 0, pos 8908800, cdp_page_count 150 00000020:00200000:0.0:1769074789.282604:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 87136, offset 0, pos 8912896, cdp_page_count 150 00000020:00200000:0.0:1769074789.282606:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 83040, offset 0, pos 8916992, cdp_page_count 150 00000020:00200000:0.0:1769074789.282609:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 78944, offset 0, pos 8921088, cdp_page_count 150 00000020:00200000:0.0:1769074789.282612:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 74848, offset 0, pos 8925184, cdp_page_count 150 00000020:00200000:0.0:1769074789.282615:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 70752, offset 0, pos 8929280, cdp_page_count 150 00000020:00200000:0.0:1769074789.282618:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 66656, offset 0, pos 8933376, cdp_page_count 150 00000020:00200000:0.0:1769074789.282621:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 62560, offset 0, pos 8937472, cdp_page_count 150 00000020:00200000:0.0:1769074789.282623:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 58464, offset 0, pos 8941568, cdp_page_count 150 00000020:00200000:0.0:1769074789.282626:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 54368, offset 0, pos 8945664, cdp_page_count 150 00000020:00200000:0.0:1769074789.282629:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 50272, offset 0, pos 8949760, cdp_page_count 150 00000020:00200000:0.0:1769074789.282632:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 46176, offset 0, pos 8953856, cdp_page_count 150 00000020:00200000:0.0:1769074789.282635:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 42080, offset 0, pos 8957952, cdp_page_count 150 00000020:00200000:0.0:1769074789.282638:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 37984, offset 0, pos 8962048, cdp_page_count 150 00000020:00200000:0.0:1769074789.282641:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 33888, offset 0, pos 8966144, cdp_page_count 150 00000020:00200000:0.0:1769074789.282645:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 29792, offset 0, pos 8970240, cdp_page_count 150 00000020:00200000:0.0:1769074789.282649:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 25696, offset 0, pos 8974336, cdp_page_count 150 00000020:00200000:0.0:1769074789.282652:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 21600, offset 0, pos 8978432, cdp_page_count 150 00000020:00200000:0.0:1769074789.282655:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 17504, offset 0, pos 8982528, cdp_page_count 150 00000020:00200000:0.0:1769074789.282658:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 13408, offset 0, pos 8986624, cdp_page_count 150 00000020:00200000:0.0:1769074789.282661:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 9312, offset 0, pos 8990720, cdp_page_count 150 00000020:00200000:0.0:1769074789.282665:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 5216, offset 0, pos 8994816, cdp_page_count 150 00000020:00200000:0.0:1769074789.282669:0:2406:0:(cl_io.c:1643:__ll_dio_user_copy()) count 1120, offset 0, pos 8998912, cdp_page_count 150 00000020:00200000:0.0:1769074789.282683:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000008:00010000:0.0:1769074789.282691:0:2406:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b0347fb6600 00000100:00100000:0.0:1769074789.282726:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616952064:192.168.204.160@tcp:3:lfs.0 00000080:00200000:0.0:1769074789.307092:0:63068:0:(file.c:2109:ll_file_io_generic()) Range unlock [0, 2197] 00000080:00200000:0.0:1769074789.307116:0:63068:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 32 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074789.307123:0:63068:0:(file.c:2142:ll_file_io_generic()) f200b.sanity-flr: 1 io complete with rc: 0, result: 9000032, restart: 0 00000020:00200000:0.0:1769074789.307127:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074789.307134:0:63068:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 9000032 00000080:00200080:0.0:1769074789.307152:0:63068:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8) cmd=400866b1 arg=0 00000080:00200080:0.0:1769074789.626670:0:63068:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8) cmd=400866b1 arg=3 00000080:00200008:0.0:1769074789.626691:0:63068:0:(file.c:6252:ll_getattr_dentry()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074789.626700:0:63068:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8),name=f200b.sanity-flr 00000002:00010000:0.0:1769074789.626736:0:63068:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x1fb:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074789.626757:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074789.626774:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074789.626783:0:63068:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074789.626793:0:63068:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x1fb:0x0](ffff9b0355ce63c8) 00000080:00010000:0.0:1769074789.626798:0:63068:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x1fb:0x0](ffff9b0355ce63c8) for lock 0xb7d6bf619879c2dd 00000080:00010000:0.0:1769074789.626806:0:63068:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c2dd from it ffffb43642fafcd8 00010000:00010000:0.0:1769074789.626809:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074789.626828:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035045ca00/0xb7d6bf619879c2dd lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x1fb:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd5461d expref: -99 pid: 63062 timeout: 0 lvb_type: 3 00000080:00200008:0.0:1769074789.626856:0:63068:0:(file.c:2474:do_file_read_iter()) START file f200b.sanity-flr:[0x200000405:0x1fb:0x0], ppos: 9000032, count: 67108864 00000080:00200000:0.0:1769074789.626873:0:63068:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 32 restore needed 0 00020000:00200000:0.0:1769074789.626888:0:63068:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074789.626891:0:63068:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:0.0:1769074789.626905:0:63068:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 32 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074789.626912:0:63068:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:0.0:1769074789.626916:0:63068:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x1fb:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 3 00020000:00200000:0.0:1769074789.626919:0:63068:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1769074789.626921:0:63068:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fb:0x0] 00000008:00010000:0.0:1769074789.626946:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.626950:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348df6e30, osc lock ffff9b0345a6d8f0, flags 0x400000400001002 00000008:00010000:0.0:1769074789.626958:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.626960:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348df6e70, osc lock ffff9b0345a6c000, flags 0x400000400001002 00000008:00010000:0.0:1769074789.626965:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.626967:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348df6eb0, osc lock ffff9b0345a6d430, flags 0x400000400001002 00000008:00010000:0.0:1769074789.626972:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.626973:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348df6ef0, osc lock ffff9b0345a6c4c0, flags 0x400000400001002 00000008:00010000:0.0:1769074789.626978:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.626979:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348df6f30, osc lock ffff9b0345a6d0a0, flags 0x400000400001002 00000008:00010000:0.0:1769074789.626986:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.626988:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348df6f70, osc lock ffff9b0345a6c720, flags 0x400000400001002 00000008:00010000:0.0:1769074789.626993:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.626995:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348df6fb0, osc lock ffff9b0345a6d690, flags 0x400000400001002 00010000:00010000:0.0:1769074789.627015:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1c00/0xb7d6bf619879c31c lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2b9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546e8 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627023:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 2097151) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1c00/0xb7d6bf619879c31c lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546e8 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.627034:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.627038:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1c00/0xb7d6bf619879c31c lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd546e8 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627045:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1c00/0xb7d6bf619879c31c lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546e8 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627060:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c323 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546ef expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627067:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 3145727) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c323 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546ef expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.627073:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.627076:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c323 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd546ef expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627083:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c323 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546ef expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627095:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c32a lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546f6 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627103:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 3145727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c32a lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546f6 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.627109:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.627112:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c32a lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd546f6 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627119:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c32a lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546f6 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627130:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c331 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2be:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546fd expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627137:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c331 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546fd expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.627144:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.627146:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c331 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd546fd expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627153:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c331 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd546fd expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627165:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd0a00/0xb7d6bf619879c338 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54704 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627177:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (3145728 4194303) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd0a00/0xb7d6bf619879c338 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54704 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.627184:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.627187:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd0a00/0xb7d6bf619879c338 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54704 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627195:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd0a00/0xb7d6bf619879c338 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54704 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627207:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c33f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bf:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5470b expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627214:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (4194304 16777215) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c33f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5470b expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.627220:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.627223:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c33f lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5470b expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627229:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c33f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5470b expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627267:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214600/0xb7d6bf619879c27b lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x2c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54567 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627280:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (4194304 16777215) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214600/0xb7d6bf619879c27b lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x2c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54567 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.627288:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074789.627290:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214600/0xb7d6bf619879c27b lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54567 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.627297:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214600/0xb7d6bf619879c27b lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54567 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.627304:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074789.627311:0:63068:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 35136 00000020:00010000:0.0:1769074789.627316:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314d918 (ffff9b03531526c8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074789.627319:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314d918 (ffff9b03531526c8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.627322:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348df6e30 (ffff9b0353152cf8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.627325:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348df6e70 (ffff9b0342b7d998/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.627328:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348df6eb0 (ffff9b0342b7db48/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.627330:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348df6ef0 (ffff9b0342b7d488/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.627333:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348df6f30 (ffff9b0342b7dbd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.627335:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348df6f70 (ffff9b0342b7d638/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.627337:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348df6fb0 (ffff9b0342b7d008/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.627339:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314d918 (ffff9b03531526c8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.627342:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348df6e30 (ffff9b03531526c8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.627346:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348df6e70 (ffff9b03531526c8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.627348:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348df6eb0 (ffff9b03531526c8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.627350:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348df6ef0 (ffff9b03531526c8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.627353:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348df6f30 (ffff9b03531526c8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.627354:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348df6f70 (ffff9b03531526c8/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.627357:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348df6fb0 (ffff9b03531526c8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074789.627373:0:63068:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/1, layout version 32 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074789.627378:0:63068:0:(file.c:2367:file_read_confine_iter()) f200b.sanity-flr: read [9000032, 76108896] over eof, kms 9000032, file_size 9000032. 00000080:00200000:0.0:1769074789.627387:0:63068:0:(file.c:1994:ll_file_io_generic()) f200b.sanity-flr: read ppos: 9000032, bytes: 67108864 00000080:00200000:0.0:1769074789.627397:0:63068:0:(file.c:1759:ll_io_set_mirror()) f200b.sanity-flr: desiginated mirror: 3 00000020:00200000:0.0:1769074789.627408:0:63068:0:(cl_io.c:214:cl_io_rw_init()) header@0000000006fe93a2[0x4, 2, [0x200000405:0x1fb:0x0] hash] 00000020:00200000:0.0:1769074789.627410:0:63068:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [9000032, 76108896) 0 0 00000080:00200000:0.0:1769074789.627414:0:63068:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 32 restore needed 0 00000080:00200000:0.0:1769074789.629545:0:63068:0:(file.c:2072:ll_file_io_generic()) Range lock [2197, 18581] 00020000:00200000:0.0:1769074789.629561:0:63068:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 2 chunk: [9000032, 12582912) 76108896, 3582880 00020000:00200000:0.0:1769074789.629566:0:63068:0:(lov_io.c:860:lov_io_iter_init()) component[12] flags 0x10 00000080:00200000:0.0:1769074789.629590:0:63068:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [2197, 3071] 00000008:00010000:0.0:1769074789.629611:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1769074789.629616:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df0b0, osc lock ffff9b0345a6d690, flags 0x0 00000080:00200000:0.0:1769074789.629625:0:63068:0:(vvp_io.c:827:vvp_io_read_start()) f200b.sanity-flr: read [9000032, 12582912) 00000001:00010000:0.0:1769074789.629639:0:63068:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fb:0x0] 00000008:00010000:0.0:1769074789.629649:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.629652:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0342ed1e30, osc lock ffff9b0345a6c720, flags 0x400001002 00000008:00010000:0.0:1769074789.629657:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.629659:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0342ed1e70, osc lock ffff9b0345a6d0a0, flags 0x400001002 00000008:00010000:0.0:1769074789.629662:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.629664:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0342ed1eb0, osc lock ffff9b0345a6c4c0, flags 0x400001002 00000008:00010000:0.0:1769074789.629669:0:63068:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074789.629671:0:63068:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0342ed1ef0, osc lock ffff9b0345a6d430, flags 0x400001002 00010000:00010000:0.0:1769074789.629732:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c2c00/0xb7d6bf619879c346 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.629743:0:63068:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c2c00/0xb7d6bf619879c346 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->4194303] (req 0->4194303) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.629808:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c2a00/0xb7d6bf619879c34d lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.629828:0:63068:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c2a00/0xb7d6bf619879c34d lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [4194304->8388607] (req 4194304->8388607) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.629852:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3e00/0xb7d6bf619879c354 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x2c4:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.629860:0:63068:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400001000 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3e00/0xb7d6bf619879c354 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x2c4:0x0].0x0 rrc: 2 type: EXT [4194304->33554431] (req 4194304->33554431) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.629876:0:63068:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e8400/0xb7d6bf619879c29e lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2c0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54598 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.629886:0:63068:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (4194304 33554431) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e8400/0xb7d6bf619879c29e lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54598 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074789.629901:0:63068:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 4194304, setting rss=4194304; kms=4194304, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e8400/0xb7d6bf619879c29e lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54598 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.629912:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00010000:00010000:0.0:1769074789.629916:0:63068:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e8400/0xb7d6bf619879c29e lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54598 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074789.629926:0:63068:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e8400/0xb7d6bf619879c29e lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54598 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074789.629943:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000080:00200000:0.0:1769074789.638785:0:63068:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fb:0x0] updating i_size 9000032 i_blocks 52696 00000020:00010000:0.0:1769074789.638793:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035312bd98 (ffff9b0342b7dd88/0) at cl_lock_release():245 00000020:00010000:0.0:1769074789.638796:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035312bd98 (ffff9b0342b7dd88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.638802:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0342ed1e30 (ffff9b0342b7d008/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.638806:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0342ed1e70 (ffff9b0342b7d638/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.638808:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0342ed1eb0 (ffff9b03531526c8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.638821:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0342ed1ef0 (ffff9b0342b7dbd8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.638824:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035312bd98 (ffff9b0342b7dd88/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.638827:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0342ed1e30 (ffff9b0342b7dd88/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.638830:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0342ed1e70 (ffff9b0342b7dd88/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.638832:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0342ed1eb0 (ffff9b0342b7dd88/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.638835:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0342ed1ef0 (ffff9b0342b7dd88/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074789.638843:0:63068:0:(vvp_io.c:886:vvp_io_read_start()) count:3582880 iocb pos:9000032 00000080:00200000:0.0:1769074789.638849:0:63068:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8), size=3582880 (max 603979776), offset=9000032=0x895460, pages 875 (max 147456), lockless, parallel, unaligned 00000020:00010000:0.0:1769074789.638865:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b034396ed58 (ffff9b0342b7dd88/0) at cl_lock_release():245 00000020:00010000:0.0:1769074789.638868:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b034396ed58 (ffff9b0342b7dd88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.638870:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df0b0 (ffff9b03531526c8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074789.638873:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b034396ed58 (ffff9b0342b7dd88/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074789.638875:0:63068:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df0b0 (ffff9b0342b7dd88/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074789.638882:0:63068:0:(vvp_io.c:518:vvp_io_advance()) advancing 0 bytes 00000020:00200000:0.0:1769074789.638889:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074789.638893:0:63068:0:(file.c:2109:ll_file_io_generic()) Range unlock [2197, 18581] 00000080:00200000:0.0:1769074789.638904:0:63068:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fb:0x0] ignore/verify layout 0/0, layout version 32 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074789.638908:0:63068:0:(file.c:2142:ll_file_io_generic()) f200b.sanity-flr: 1 io complete with rc: 0, result: 0, restart: 0 00000020:00200000:0.0:1769074789.638911:0:63068:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074789.638917:0:63068:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 0 00000080:00200080:0.0:1769074789.638931:0:63068:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fb:0x0](ffff9b0355ce63c8) cmd=400866b1 arg=0 00000100:00100000:0.0:1769074790.485514:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007616952960:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074790.485632:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616953216:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074790.485670:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616953472:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074790.485706:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074790.485712:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616953088:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074790.485733:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616953344:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074790.490391:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616953216:192.168.204.160@tcp:400:kworker.0 00010000:00010000:0.0:1769074792.851873:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074794.899367:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074795.612559:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616954624:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074795.612582:0:2406:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9b0342ad2800: committing for last_committed 17179874587 gen 1 00000100:00100000:0.0:1769074795.612618:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616954880:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074795.612633:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007616954496:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074795.612642:0:2406:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9b0342ad2800: committing for last_committed 21474841256 gen 1 00000100:00100000:0.0:1769074795.612659:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616954752:192.168.204.160@tcp:400:kworker.0 00000001:02000400:0.0:1769074797.671118:0:63636:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 200c: layout change racing with open: LOVEA changes ========================================================== 04:39:55 (1769074795) 00000100:00100000:0.0:1769074797.950859:0:63659:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8e00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:a7835f93-1e5d-43a4-8110-91f072d9ae65:63659:1855007616955136:192.168.204.160@tcp:501:mount.lustre.0 00000100:00100000:0.0:1769074797.950931:0:63659:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job mount.lustre:a7835f93-1e5d-43a4-8110-91f072d9ae65:63659:1855007616955264:192.168.204.160@tcp:503:mount.lustre.0 00000100:00100000:0.0:1769074797.951013:0:63659:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0356956480 going to sleep for 11 seconds 00000100:00100000:0.0:1769074797.954189:0:63659:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job mount.lustre:a7835f93-1e5d-43a4-8110-91f072d9ae65:63659:1855007616955264:192.168.204.160@tcp:503:mount.lustre.0 00010000:00010000:0.0:1769074797.973009:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074797.983874:0:63665:0:(ldlm_pool.c:924:ldlm_pool_init()) Lock pool ldlm-pool-lustre-MDT0001-mdc-ffff9b035d92c800-6 is initialized 00000040:01000000:0.0:1769074797.984086:0:63665:0:(llog_obd.c:195:llog_setup()) obd lustre-MDT0001-mdc-ffff9b035d92c800 ctxt 13 is initialized 00000020:00000080:0.0:1769074797.984134:0:63665:0:(obd_config.c:761:class_setup()) finished setup of obd lustre-MDT0001-mdc-ffff9b035d92c800 (uuid 2274e3cf-6162-4197-bb1e-d97abc9b4bb9) 00000020:01000000:0.0:1769074797.984149:0:63665:0:(obd_config.c:1903:class_config_llog_handler()) cmd cf014, instance name: lustre-clilmv-ffff9b035d92c800 00000020:00000080:0.0:1769074797.984153:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf014 00800000:01000000:0.0:1769074797.984179:0:63665:0:(lmv_obd.c:425:lmv_add_target()) Target uuid: lustre-MDT0001_UUID. index 1 00000020:01000000:0.0:1769074797.984222:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1769074797.984226:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1769074797.984227:0:63665:0:(obd_config.c:1477:class_process_config()) marker 16 (0x2) lustre-MDT0001 add mdc 00000020:01000000:0.0:1769074797.984230:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:0.0:1769074797.984262:0:63665:0:(obd_config.c:1797:class_config_llog_handler()) SKIP #18 00000020:01000000:0.0:1769074797.984265:0:63665:0:(obd_config.c:1823:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1769074797.984267:0:63665:0:(obd_config.c:1823:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1769074797.984268:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:0.0:1769074797.984271:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1769074797.984273:0:63665:0:(obd_config.c:1477:class_process_config()) marker 18 (0x6) lustre sys.timeout 00000020:01000000:0.0:1769074797.984275:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:0.0:1769074797.984281:0:63665:0:(obd_mount.c:939:lustre_check_exclusion()) Check exclusion lustre-OST0000 (0) in 0 of 192.168.204.160@tcp:/lustre 00000020:00000080:0.0:1769074797.984284:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1769074797.984286:0:63665:0:(obd_config.c:1477:class_process_config()) marker 24 (0x1) lustre-OST0000 add osc 00000020:00000080:0.0:1769074797.984289:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1769074797.984295:0:63665:0:(obd_config.c:1401:class_process_config()) adding mapping from uuid 192.168.204.160@tcp to nid 0x20000c0a8cca0 (192.168.204.160@tcp) 00000020:01000000:0.0:1769074797.984324:0:63665:0:(obd_config.c:1903:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0000-osc-ffff9b035d92c800 00000020:00000080:0.0:1769074797.984326:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1769074797.984339:0:63665:0:(genops.c:412:class_newdev()) Allocate new device lustre-OST0000-osc-ffff9b035d92c800 (ffff9b0342f131b0) 00000020:00000080:0.0:1769074797.984354:0:63665:0:(obd_config.c:657:class_attach()) OBD: dev 11 attached type osc with refcount 2 00000020:01000000:0.0:1769074797.984359:0:63665:0:(obd_config.c:1903:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0000-osc-ffff9b035d92c800 00000020:00000080:0.0:1769074797.984363:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf003 00010000:00080000:0.0:1769074797.984435:0:63665:0:(ldlm_lib.c:102:import_set_conn()) imp ffff9b03593e0800@lustre-OST0000-osc-ffff9b035d92c800: add connection 192.168.204.160@tcp at head 00010000:00010000:0.0:1769074797.984590:0:63665:0:(ldlm_pool.c:924:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0000-osc-ffff9b035d92c800-7 is initialized 00000020:00000080:0.0:1769074797.984797:0:63665:0:(obd_config.c:761:class_setup()) finished setup of obd lustre-OST0000-osc-ffff9b035d92c800 (uuid 2274e3cf-6162-4197-bb1e-d97abc9b4bb9) 00000020:01000000:0.0:1769074797.984803:0:63665:0:(obd_config.c:1903:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffff9b035d92c800 00000020:00000080:0.0:1769074797.984805:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf00d 00020000:01000000:0.0:1769074797.984818:0:63665:0:(lov_obd.c:469:lov_add_target()) uuid:lustre-OST0000_UUID idx:0 gen:1 active:1 00020000:01000000:0.0:1769074797.984824:0:63665:0:(lov_obd.c:498:lov_add_target()) lustre-clilov-ffff9b035d92c800: Adding target lustre-OST0000_UUID 00020000:01000000:0.0:1769074797.984829:0:63665:0:(lov_obd.c:520:lov_add_target()) idx=0 ltd_gen=1 ld_tgt_count=1 00000020:01000000:0.0:1769074797.984839:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1769074797.984843:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1769074797.984844:0:63665:0:(obd_config.c:1477:class_process_config()) marker 24 (0x2) lustre-OST0000 add osc 00000020:01000000:0.0:1769074797.984847:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:0.0:1769074797.984849:0:63665:0:(obd_config.c:1797:class_config_llog_handler()) SKIP #25 00000020:01000000:0.0:1769074797.984851:0:63665:0:(obd_config.c:1823:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1769074797.984852:0:63665:0:(obd_config.c:1823:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1769074797.984853:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:0.0:1769074797.984855:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1769074797.984857:0:63665:0:(obd_config.c:1477:class_process_config()) marker 25 (0x6) lustre sys.timeout 00000020:01000000:0.0:1769074797.984859:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:0.0:1769074797.984862:0:63665:0:(obd_mount.c:939:lustre_check_exclusion()) Check exclusion lustre-OST0001 (1) in 0 of 192.168.204.160@tcp:/lustre 00000020:00000080:0.0:1769074797.984865:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1769074797.984866:0:63665:0:(obd_config.c:1477:class_process_config()) marker 32 (0x1) lustre-OST0001 add osc 00000020:00000080:0.0:1769074797.984869:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1769074797.984872:0:63665:0:(obd_config.c:1401:class_process_config()) adding mapping from uuid 192.168.204.160@tcp to nid 0x20000c0a8cca0 (192.168.204.160@tcp) 00000020:01000000:0.0:1769074797.984884:0:63665:0:(obd_config.c:1903:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0001-osc-ffff9b035d92c800 00000020:00000080:0.0:1769074797.984886:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1769074797.984892:0:63665:0:(genops.c:412:class_newdev()) Allocate new device lustre-OST0001-osc-ffff9b035d92c800 (ffff9b0342f14a88) 00000020:00000080:0.0:1769074797.984903:0:63665:0:(obd_config.c:657:class_attach()) OBD: dev 12 attached type osc with refcount 2 00000020:01000000:0.0:1769074797.984908:0:63665:0:(obd_config.c:1903:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0001-osc-ffff9b035d92c800 00000020:00000080:0.0:1769074797.984911:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf003 00010000:00080000:0.0:1769074797.984957:0:63665:0:(ldlm_lib.c:102:import_set_conn()) imp ffff9b03593e2800@lustre-OST0001-osc-ffff9b035d92c800: add connection 192.168.204.160@tcp at head 00010000:00010000:0.0:1769074797.985138:0:63665:0:(ldlm_pool.c:924:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0001-osc-ffff9b035d92c800-8 is initialized 00000020:00000080:0.0:1769074797.985328:0:63665:0:(obd_config.c:761:class_setup()) finished setup of obd lustre-OST0001-osc-ffff9b035d92c800 (uuid 2274e3cf-6162-4197-bb1e-d97abc9b4bb9) 00000020:01000000:0.0:1769074797.985333:0:63665:0:(obd_config.c:1903:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffff9b035d92c800 00000020:00000080:0.0:1769074797.985336:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf00d 00020000:01000000:0.0:1769074797.985346:0:63665:0:(lov_obd.c:469:lov_add_target()) uuid:lustre-OST0001_UUID idx:1 gen:1 active:1 00020000:01000000:0.0:1769074797.985351:0:63665:0:(lov_obd.c:498:lov_add_target()) lustre-clilov-ffff9b035d92c800: Adding target lustre-OST0001_UUID 00020000:01000000:0.0:1769074797.985353:0:63665:0:(lov_obd.c:520:lov_add_target()) idx=1 ltd_gen=1 ld_tgt_count=2 00000020:01000000:0.0:1769074797.985361:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1769074797.985364:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1769074797.985365:0:63665:0:(obd_config.c:1477:class_process_config()) marker 32 (0x2) lustre-OST0001 add osc 00000020:01000000:0.0:1769074797.985367:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1769074797.985370:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1769074797.985371:0:63665:0:(obd_config.c:1477:class_process_config()) marker 33 (0x1) lustre sys.timeout 00000020:00000080:0.0:1769074797.985373:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf009 00000020:00000080:0.0:1769074797.985375:0:63665:0:(obd_config.c:1451:class_process_config()) changing lustre timeout from 20 to 20 00000020:01000000:0.0:1769074797.985376:0:63665:0:(obd_config.c:1788:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1769074797.985378:0:63665:0:(obd_config.c:1389:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1769074797.985379:0:63665:0:(obd_config.c:1477:class_process_config()) marker 33 (0x2) lustre sys.timeout 00000100:00100000:0.0:1769074797.985418:0:63665:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job llog_process_th:a7835f93-1e5d-43a4-8110-91f072d9ae65:63665:1855007616955520:192.168.204.160@tcp:502:llog_process_th.0 00000100:00100000:0.0:1769074797.985610:0:63665:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0356956b40 going to sleep for 11 seconds 00000020:01000000:0.0:1769074798.014191:0:63659:0:(obd_config.c:2043:class_config_parse_llog()) Processed log lustre-client gen 1-51 (rc=0) 10000000:01000000:0.0:1769074798.014209:0:63659:0:(mgc_request.c:1889:mgc_process_log()) MGC192.168.204.160@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:0.0:1769074798.014226:0:63659:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.160@tcp lock: ffff9b0359292400/0xb7d6bf619879c370 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x7fe2323adfd54790 expref: -99 pid: 63659 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074798.014265:0:63659:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.160@tcp lock: ffff9b0359292400/0xb7d6bf619879c370 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x7fe2323adfd54790 expref: -99 pid: 63659 timeout: 0 lvb_type: 0 10000000:01000000:0.0:1769074798.014273:0:63659:0:(mgc_request.c:1812:mgc_process_log()) Process log lustre-cliir-ffff9b035d92c800 from 1 10000000:01000000:0.0:1769074798.014275:0:63659:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:0.0:1769074798.014310:0:63659:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.160@tcp lock: ffff9b03466c2400/0xb7d6bf619879c3a1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 63659 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074798.014317:0:63659:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.160@tcp lock: ffff9b03466c2400/0xb7d6bf619879c3a1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 63659 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074798.014324:0:63659:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.160@tcp lock: ffff9b03466c2400/0xb7d6bf619879c3a1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 63659 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074798.014341:0:63659:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job mount.lustre:a7835f93-1e5d-43a4-8110-91f072d9ae65:63659:1855007616955776:192.168.204.160@tcp:101:mount.lustre.0 00000100:00100000:0.0:1769074798.014425:0:63659:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0356956300 going to sleep for 11 seconds 00000100:00080000:0.0:1769074798.098951:0:2406:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff9b0342ce9c00 x1855007616956800/t0(0) o41->lustre-MDT0000-mdc-ffff9b035d92c800@192.168.204.160@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000080:00200008:0.0:1769074798.399159:0:63691:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074798.399170:0:63691:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074798.399193:0:63691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074798.399210:0:63691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c35b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5475f expref: -99 pid: 63493 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074798.399229:0:63691:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c35b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5475f expref: -99 pid: 63493 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074798.399457:0:63691:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074798.399464:0:63691:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074798.399467:0:63691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c35b 00000080:00010000:0.0:1769074798.399476:0:63691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c35b from it ffffb43642277cb8 00010000:00010000:0.0:1769074798.399480:0:63691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c35b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5475f expref: -99 pid: 63493 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074798.399489:0:63691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c35b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5475f expref: -99 pid: 63493 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074798.399709:0:63691:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074798.399719:0:63691:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074798.399726:0:63691:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c35b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5475f expref: -99 pid: 63493 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074798.399735:0:63691:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c35b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5475f expref: -99 pid: 63493 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074798.399743:0:63691:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074798.399747:0:63691:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074798.399750:0:63691:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c35b 00000080:00010000:0.0:1769074798.399754:0:63691:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c35b from it ffffb43642277bb0 00010000:00010000:0.0:1769074798.399756:0:63691:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c35b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5475f expref: -99 pid: 63493 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074798.399763:0:63691:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c35b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5475f expref: -99 pid: 63493 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074798.399771:0:63691:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074798.399778:0:63691:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 100000 00000080:00200080:0.0:1769074798.399907:0:63691:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=c00866a4 arg=7ffe07be2a60 00800000:00000080:0.0:1769074798.399924:0:63691:0:(lmv_obd.c:905:lmv_iocontrol()) lustre-clilmv-ffff9b0342ad2800: cmd=c00866a4 len=592 karg=ffff9b0372dca400 uarg=0000000000000000 00800000:00000004:0.0:1769074798.399930:0:63691:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff9b0342ad2800: age 1608, max_age 1791 00000100:00100000:0.0:1769074798.399975:0:63691:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:63691:1855007616958464:192.168.204.160@tcp:41:lfs.0 00000100:00100000:0.0:1769074798.479135:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625197312 00000100:00100000:0.0:1769074798.479150:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625197312 00000100:00100000:0.0:1769074798.479278:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8895 00000100:00100000:0.0:1769074798.479291:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:2274e3cf-6162-4197-bb1e-d97abc9b4bb9+5:10283:x1855007625197312:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074798.479307:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0342cd0400/0xb7d6bf619879c3d9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0x7fe2323adfd547c8 expref: -99 pid: 63691 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074798.479392:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074798.479400:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:2274e3cf-6162-4197-bb1e-d97abc9b4bb9+5:10283:x1855007625197312:12345-192.168.204.160@tcp:104: Request processed in 112us (3387us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074798.479407:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8895 00010000:00010000:0.0:1769074798.710563:0:60112:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 11 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074798.710569:0:60112:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b03466c2e00/0xb7d6bf6198795592 lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x48:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd4875d expref: -99 pid: 58467 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074798.710604:0:60112:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b03531e3800 x1855007615186432/t4294968343(4294968343) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074630 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074798.710613:0:60112:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074798.710617:0:60112:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3aa00 x1855007616961024/t0(0) o35->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2192 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074798.710630:0:60112:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_05:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:60112:1855007616961024:192.168.204.160@tcp:35:ldlm_bl.0 00000100:00100000:0.0:1769074798.710657:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0356956e40 going to sleep for 11 seconds 00010000:00010000:0.0:1769074798.713130:0:63697:0:(ldlm_resource.c:158:lru_size_store()) dropping all unused locks from namespace lustre-MDT0001-mdc-ffff9b035d92c800 00010000:00010000:0.0:1769074798.839149:0:46917:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 16 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074798.839160:0:46917:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c3e7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800028400000000 nid: local remote: 0x7fe2323adfd547f2 expref: -99 pid: 63692 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074798.839189:0:46917:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### extent: 0000000027298a34 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c3e7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800029400000000 nid: local remote: 0x7fe2323adfd547f2 expref: -99 pid: 63692 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074798.839209:0:46917:0:(osc_cache.c:1103:osc_extent_make_ready()) ### extent: 0000000027298a34 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c3e7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800029400000000 nid: local remote: 0x7fe2323adfd547f2 expref: -99 pid: 63692 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.839758:0:46917:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c3e7 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800029400000000 nid: local remote: 0x7fe2323adfd547f2 expref: -99 pid: 63692 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074798.840666:0:46917:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a39180 - object 0x280000401:705 offset 0<>2097152 00000100:00100000:0.0:1769074798.840752:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074798.840773:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616961664:192.168.204.160@tcp:4:dd.0 00010000:00010000:0.0:1769074798.841111:0:63703:0:(ldlm_resource.c:158:lru_size_store()) dropping all unused locks from namespace lustre-OST0000-osc-ffff9b035d92c800 00010000:00010000:0.0:1769074798.841153:0:63703:0:(ldlm_resource.c:158:lru_size_store()) dropping all unused locks from namespace lustre-OST0001-osc-ffff9b0342ad2800 00010000:00010000:0.0:1769074798.841195:0:63703:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 21/21 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1769074798.841206:0:60220:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 21 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074798.841213:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3e00/0xb7d6bf619879c354 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x28400010000 nid: local remote: 0x7fe2323adfd5472e expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841256:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c2c00/0xb7d6bf619879c346 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x28400010000 nid: local remote: 0x7fe2323adfd54727 expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841267:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c2a00/0xb7d6bf619879c34d lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x28400010000 nid: local remote: 0x7fe2323adfd54735 expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841274:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214600/0xb7d6bf619879c27b lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd54567 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841282:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd0a00/0xb7d6bf619879c338 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x28400010000 nid: local remote: 0x7fe2323adfd54704 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841291:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c32a lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x28400010000 nid: local remote: 0x7fe2323adfd546f6 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841300:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x28400010000 nid: local remote: 0x7fe2323adfd54694 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841308:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x28400010000 nid: local remote: 0x7fe2323adfd54686 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841316:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c2eb lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x28400010000 nid: local remote: 0x7fe2323adfd54671 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841324:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035046d000/0xb7d6bf619878fb48 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x800028400020000 nid: local remote: 0x7fe2323adfd396e0 expref: -99 pid: 57541 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074798.841340:0:60220:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x47:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074798.841501:0:60220:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x47:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1769074798.841516:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0343270e00/0xb7d6bf619879570c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x28400020000 nid: local remote: 0x7fe2323adfd48cab expref: -99 pid: 59259 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841525:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0343271200/0xb7d6bf6198795705 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x28400020000 nid: local remote: 0x7fe2323adfd48ca4 expref: -99 pid: 59259 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841532:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0343271e00/0xb7d6bf61987956fe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->4194303) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd48c96 expref: -99 pid: 59259 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841545:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035046c000/0xb7d6bf6198795553 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x28400010000 nid: local remote: 0x7fe2323adfd48699 expref: -99 pid: 58464 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841552:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350989800/0xb7d6bf61987860b5 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x89:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028480000000 nid: local remote: 0x7fe2323adfd1bd25 expref: -99 pid: 55379 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074798.841561:0:60220:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x25:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074798.841584:0:60220:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x25:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1769074798.841594:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0353174400/0xb7d6bf6198785fc7 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd1ba1c expref: -99 pid: 54786 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841603:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0353174000/0xb7d6bf6198785e70 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x85:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x28400020000 nid: local remote: 0x7fe2323adfd1b61e expref: -99 pid: 54192 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841611:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0353175c00/0xb7d6bf6198785e69 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x86:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400000000 nid: local remote: 0x7fe2323adfd1b609 expref: -99 pid: 54192 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074798.841619:0:60220:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x21:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074798.841624:0:60220:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x21:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1769074798.841630:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03429b5a00/0xb7d6bf6198785ce8 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x28400010000 nid: local remote: 0x7fe2323adfd1b17f expref: -99 pid: 53562 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841638:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03429b4200/0xb7d6bf6198785a6b lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0x2c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400020000 nid: local remote: 0x7fe2323adfd1a9a6 expref: -99 pid: 52857 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074798.841650:0:60220:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x3a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074798.841654:0:60220:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x3a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1769074798.841674:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035045dc00/0xb7d6bf6198784b84 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x84:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd18151 expref: -99 pid: 51962 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841705:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c3e00/0xb7d6bf619879c354 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x2c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd5472e expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841713:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c2c00/0xb7d6bf619879c346 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4194303) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd54727 expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841719:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03466c2a00/0xb7d6bf619879c34d lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->8388607) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd54735 expref: -99 pid: 63068 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841725:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0348214600/0xb7d6bf619879c27b lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd54567 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841732:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd0a00/0xb7d6bf619879c338 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd54704 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841739:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1800/0xb7d6bf619879c32a lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd546f6 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841745:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293e00/0xb7d6bf619879c307 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd54694 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841750:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0359293200/0xb7d6bf619879c300 lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd54686 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841757:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c2eb lrc: 1/0,0 mode: --/PR res: [0x2c0000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd54671 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841763:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035046d000/0xb7d6bf619878fb48 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->8191) gid 0 flags: 0x804869400020020 nid: local remote: 0x7fe2323adfd396e0 expref: -99 pid: 57541 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841770:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0343270e00/0xb7d6bf619879570c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2b2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x4829400020020 nid: local remote: 0x7fe2323adfd48cab expref: -99 pid: 59259 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841777:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0343271200/0xb7d6bf6198795705 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2b1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4829400020020 nid: local remote: 0x7fe2323adfd48ca4 expref: -99 pid: 59259 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841783:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0343271e00/0xb7d6bf61987956fe lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2b3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->4194303) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd48c96 expref: -99 pid: 59259 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841788:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035046c000/0xb7d6bf6198795553 lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd48699 expref: -99 pid: 58464 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841794:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0350989800/0xb7d6bf61987860b5 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x89:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869480000020 nid: local remote: 0x7fe2323adfd1bd25 expref: -99 pid: 55379 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841800:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0353174400/0xb7d6bf6198785fc7 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0x32:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd1ba1c expref: -99 pid: 54786 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841806:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0353174000/0xb7d6bf6198785e70 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x85:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400020020 nid: local remote: 0x7fe2323adfd1b61e expref: -99 pid: 54192 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841817:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0353175c00/0xb7d6bf6198785e69 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x86:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400000020 nid: local remote: 0x7fe2323adfd1b609 expref: -99 pid: 54192 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841822:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03429b5a00/0xb7d6bf6198785ce8 lrc: 1/0,0 mode: --/PR res: [0x2c0000400:0x31:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd1b17f expref: -99 pid: 53562 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841829:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b03429b4200/0xb7d6bf6198785a6b lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0x2c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400020020 nid: local remote: 0x7fe2323adfd1a9a6 expref: -99 pid: 52857 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841835:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035045dc00/0xb7d6bf6198784b84 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x84:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd18151 expref: -99 pid: 51962 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.841840:0:60220:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 21 locks packed 00000100:00100000:0.0:1769074798.841860:0:60220:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_06:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:60220:1855007616961792:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074798.841913:0:60220:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0356956840 going to sleep for 11 seconds 00000008:00010000:0.0:1769074798.894083:0:2407:0:(osc_cache.c:861:osc_extent_finish()) ### extent: 0000000027298a34 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c3e7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800029400000000 nid: local remote: 0x7fe2323adfd547f2 expref: -99 pid: 63692 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074798.894538:0:2407:0:(osc_cache.c:483:__osc_extent_remove()) ### extent: 0000000027298a34 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c3e7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800029400000000 nid: local remote: 0x7fe2323adfd547f2 expref: -99 pid: 63692 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074798.894569:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616961664:192.168.204.160@tcp:4:dd.0 00000100:00100000:0.0:1769074798.905599:0:46917:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_03:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:46917:1855007616961920:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074798.905705:0:46917:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0356956540 going to sleep for 11 seconds 00000100:00100000:0.0:1769074798.921579:0:46917:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job ldlm_bl_03:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:46917:1855007616961920:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074798.921619:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e9600/0xb7d6bf619879c3e7 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x804869400000020 nid: local remote: 0x7fe2323adfd547f2 expref: -99 pid: 63692 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921644:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03453e8400/0xb7d6bf619879c29e lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2c0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->33554431) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd54598 expref: -99 pid: 63053 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921665:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c33f lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2bf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 4194304->16777215) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd5470b expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921678:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c331 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2be:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 3145728->4194303) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd546fd expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921689:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c323 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2bd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->3145727) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd546ef expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921700:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1c00/0xb7d6bf619879c31c lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2b9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd546e8 expref: -99 pid: 63065 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921712:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0359292c00/0xb7d6bf619879c30e lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2bc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->8388607) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd5468d expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921723:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c3000/0xb7d6bf619879c2f9 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2bb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd5467f expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921732:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2200/0xb7d6bf619879c2f2 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2ba:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x4829400010020 nid: local remote: 0x7fe2323adfd54678 expref: -99 pid: 63062 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921747:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2600/0xb7d6bf6198795713 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2af:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->3145727) gid 0 flags: 0x4829400020020 nid: local remote: 0x7fe2323adfd48cb2 expref: -99 pid: 59259 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921757:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0343271800/0xb7d6bf61987956f7 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2ae:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd48c8f expref: -99 pid: 59259 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921768:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b03466c2000/0xb7d6bf6198795561 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x34:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400010020 nid: local remote: 0x7fe2323adfd486ae expref: -99 pid: 58464 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921778:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0353174e00/0xb7d6bf6198785fce lrc: 0/0,0 mode: --/PW res: [0x280000400:0x33:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400020020 nid: local remote: 0x7fe2323adfd1ba2a expref: -99 pid: 54786 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921788:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf6198784b99 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x84:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400020020 nid: local remote: 0x7fe2323adfd18189 expref: -99 pid: 51962 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921811:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0358aa1000/0xb7d6bf6198784a9d lrc: 0/0,0 mode: --/PR res: [0x280000400:0x24:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 2097152->18446744073709551615) gid 0 flags: 0x804869400010020 nid: local remote: 0x7fe2323adfd17eb8 expref: -99 pid: 51379 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921823:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0358aa1a00/0xb7d6bf6198784a96 lrc: 0/0,0 mode: --/PR res: [0x280000400:0x23:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->2097151) gid 0 flags: 0x804869400010020 nid: local remote: 0x7fe2323adfd17eb1 expref: -99 pid: 51379 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074798.921854:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074798.921857:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074798.921859:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074800.021353:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1769074802.916910:0:63851:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x1fe:0x0](ffff9b0355ce1148), inode mode 81a4 mask 4 00000080:00200000:0.0:1769074802.916931:0:63851:0:(xattr.c:725:ll_xattr_get()) VFS Op:inode=[0x200000405:0x1fe:0x0](ffff9b0355ce1148), xattr lov 00000080:00200008:0.0:1769074802.916966:0:63851:0:(file.c:6252:ll_getattr_dentry()) START file f200c.sanity-flr:[0x200000405:0x1fe:0x0](ffff9b0355ce1148), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074802.916974:0:63851:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x1fe:0x0](ffff9b0355ce1148),name=f200c.sanity-flr 00000002:00010000:0.0:1769074802.917011:0:63851:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x1fe:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074802.917029:0:63851:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0342cd1200/0xb7d6bf619879c411 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fe:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd548d9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074802.917044:0:63851:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0342cd1200/0xb7d6bf619879c411 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x1fe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd548d9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074802.917052:0:63851:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074802.917060:0:63851:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x1fe:0x0](ffff9b0355ce1148) 00000080:00010000:0.0:1769074802.917064:0:63851:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x1fe:0x0](ffff9b0355ce1148) for lock 0xb7d6bf619879c411 00000080:00010000:0.0:1769074802.917071:0:63851:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c411 from it ffffb43641dc3cd8 00010000:00010000:0.0:1769074802.917075:0:63851:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0342cd1200/0xb7d6bf619879c411 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x1fe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd548d9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074802.917083:0:63851:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0342cd1200/0xb7d6bf619879c411 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x1fe:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd548d9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074802.917111:0:63851:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fe:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:0.0:1769074802.917120:0:63851:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x1fe:0x0] 00000008:00010000:0.0:1769074802.917139:0:63851:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074802.917142:0:63851:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568df7b0, osc lock ffff9b0345a6d690, flags 0x400001002 00010000:00010000:0.0:1769074802.917159:0:63851:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b035d92c800 lock: ffff9b0342cd0000/0xb7d6bf619879c426 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54911 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074802.917168:0:63851:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffff9b035d92c800 lock: ffff9b0342cd0000/0xb7d6bf619879c426 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54911 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074802.917185:0:63851:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074802.917189:0:63851:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b035d92c800 lock: ffff9b0342cd0000/0xb7d6bf619879c426 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54911 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074802.917197:0:63851:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b035d92c800 lock: ffff9b0342cd0000/0xb7d6bf619879c426 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54911 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074802.917208:0:63851:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074802.917219:0:63851:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fe:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1769074802.917223:0:63851:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035312ab58 (ffff9b0342b7dbd8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074802.917226:0:63851:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035312ab58 (ffff9b0342b7dbd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074802.917229:0:63851:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568df7b0 (ffff9b0342b7d878/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074802.917869:0:63851:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035312ab58 (ffff9b0342b7dbd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074802.917875:0:63851:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568df7b0 (ffff9b0342b7dbd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074802.917887:0:63851:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fe:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1769074802.956319:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2274e3cf-6162-4197-bb1e-d97abc9b4bb9:2407:1855007616964864:192.168.204.160@tcp:23:lfs.0 00000100:00100000:0.0:1769074802.970304:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:2274e3cf-6162-4197-bb1e-d97abc9b4bb9:2407:1855007616964864:192.168.204.160@tcp:23:lfs.0 00000100:00100000:0.0:1769074803.053487:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00010000:00010000:0.0:1769074803.091431:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074803.161272:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625202304 00000100:00100000:0.0:1769074803.161284:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625202304 00000100:00100000:0.0:1769074803.161330:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8898 00000100:00100000:0.0:1769074803.161339:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:9048:x1855007625202304:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074803.161355:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b035d92c800 lock: ffff9b0365860400/0xb7d6bf619879c43b lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd5492d expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074803.161437:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074803.161443:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:9048:x1855007625202304:12345-192.168.204.160@tcp:104: Request processed in 105us (335us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074803.161450:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8898 00000100:00100000:0.0:1769074803.165865:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625202432 00000100:00100000:0.0:1769074803.165877:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625202432 00000100:00100000:0.0:1769074803.165929:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8899 00000100:00100000:0.0:1769074803.165946:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6498:x1855007625202432:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074803.165961:0:39330:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff9b035d92c800 lock: ffff9b0365861a00/0xb7d6bf619879c45e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54988 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074803.165972:0:39330:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff9b035d92c800 lock: ffff9b0365861a00/0xb7d6bf619879c45e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54988 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074803.165981:0:39330:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff9b035d92c800 lock: ffff9b0365861a00/0xb7d6bf619879c45e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54988 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074803.165991:0:39330:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff9b035d92c800 lock: ffff9b0365861a00/0xb7d6bf619879c45e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd54988 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074803.166001:0:39330:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b035d92c800 lock: ffff9b0365861a00/0xb7d6bf619879c45e lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd54988 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074803.166011:0:39330:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff9b035d92c800 lock: ffff9b0365861a00/0xb7d6bf619879c45e lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000000000 nid: local remote: 0x7fe2323adfd54988 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074803.166033:0:39330:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b0365861a00) 00000100:00100000:0.0:1769074803.166082:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6498:x1855007625202432:12345-192.168.204.160@tcp:105: Request processed in 138us (332us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074803.166088:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8899 00000100:00100000:0.0:1769074803.192464:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074803.221864:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074803.237700:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000020:00010000:0.0:1769074803.250634:0:63851:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03592838a8 (ffff9b035121f518/0) at cl_lock_release():245 00000020:00010000:0.0:1769074803.250639:0:63851:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03592838a8 (ffff9b035121f518/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074803.250645:0:63851:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6b9b0 (ffff9b0342b7d368/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074803.250656:0:63851:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b035d92c800 lock: ffff9b0365861800/0xb7d6bf619879c46c lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd549a4 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074803.250670:0:63851:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b035d92c800 lock: ffff9b0365861800/0xb7d6bf619879c46c lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd549a4 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074803.250684:0:63851:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03592838a8 (ffff9b035121f518/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074803.250688:0:63851:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6b9b0 (ffff9b035121f518/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074803.250706:0:63851:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fe:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1769074803.250719:0:63851:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file f200c.sanity-flr:[0x200000405:0x1fe:0x0](ffff9b0355ce1148) current size 2097152, valid attrs 101f0, mode 9b03, uid 4785, gid 5, new size 4294967296, atime 1769074798.000000000, mtime 1769074798.000000000, ctime 1769074803.000000000, rc 0 00000080:00200080:0.0:1769074803.250748:0:63851:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fe:0x0](ffff9b0355ce1148) cmd=c00c66f3 arg=19ed450 00010000:00010000:0.0:1769074803.250805:0:63851:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0342cd1000/0xb7d6bf619879c442 lrc: 2/0,0 mode: PW/PW res: [0x200000405:0x1fe:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7fe2323adfd5493b expref: -99 pid: 63851 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074803.250827:0:63851:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0342cd1000/0xb7d6bf619879c442 lrc: 0/0,0 mode: --/PW res: [0x200000405:0x1fe:0x0].0x0 bits 0x4/0x0 rrc: 3 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7fe2323adfd5493b expref: -99 pid: 63851 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074803.251177:0:63851:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0352a18380 x1855007616964608/t8589944031(8589944031) o101->lustre-MDT0000-mdc-ffff9b035d92c800@192.168.204.160@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074818 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074803.251196:0:63851:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b035d92c800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074803.251204:0:63851:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a39f80 x1855007616968320/t0(0) o35->lustre-MDT0000-mdc-ffff9b035d92c800@192.168.204.160@tcp:23/10 lens 504/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074803.251226:0:63851:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job lfs:2274e3cf-6162-4197-bb1e-d97abc9b4bb9:63851:1855007616968320:192.168.204.160@tcp:35:lfs.0 00000100:00100000:0.0:1769074803.251543:0:63851:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0356956300 going to sleep for 11 seconds 00000100:00100000:0.0:1769074803.255145:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625202944 00000100:00100000:0.0:1769074803.255155:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625202944 00000100:00100000:0.0:1769074803.255266:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8903 00000100:00100000:0.0:1769074803.255276:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:2274e3cf-6162-4197-bb1e-d97abc9b4bb9+5:6515:x1855007625202944:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074803.255292:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0365860c00/0xb7d6bf619879c465 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd5499d expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074803.255372:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074803.255379:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:2274e3cf-6162-4197-bb1e-d97abc9b4bb9+5:6515:x1855007625202944:12345-192.168.204.160@tcp:104: Request processed in 105us (533us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074803.255386:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8903 00000100:00100000:0.0:1769074803.278024:0:63851:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job lfs:2274e3cf-6162-4197-bb1e-d97abc9b4bb9:63851:1855007616968320:192.168.204.160@tcp:35:lfs.0 00000002:00100000:0.0:1769074803.278063:0:63851:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0352a18380 x1855007616964608/t8589944031(8589944031) o101->lustre-MDT0000-mdc-ffff9b035d92c800@192.168.204.160@tcp:12/10 lens 576/664 e 0 to 0 dl 1769074818 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1769074803.278114:0:63851:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000405:0x1fe:0x0](ffff9b0355ce1148) 00000002:00010000:0.0:1769074803.278125:0:63851:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x1fe:0x0]) in obj [0x200000405:0x1fe:0x0], intent: layout flags 0 00010000:00010000:0.0:1769074803.278173:0:63851:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 63851 initiator: MDT0 00010000:00010000:0.0:1769074803.278183:0:63851:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63851 initiator: MDT0 00010000:00010000:0.0:1769074803.278191:0:63851:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 63851 initiator: MDT0 00000100:00100000:0.0:1769074803.278210:0:63851:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job lfs:2274e3cf-6162-4197-bb1e-d97abc9b4bb9:63851:1855007616968704:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074803.278359:0:63851:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0356956300 going to sleep for 11 seconds 00000100:00100000:0.0:1769074803.293945:0:63851:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job lfs:2274e3cf-6162-4197-bb1e-d97abc9b4bb9:63851:1855007616968704:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074803.293980:0:63851:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074803.293994:0:63851:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00000002:00100000:0.0:1769074803.294008:0:63851:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9b0365a39f80 x1855007616968704/t0(0) o101->lustre-MDT0000-mdc-ffff9b035d92c800@192.168.204.160@tcp:12/10 lens 376/608 e 0 to 0 dl 1769074819 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074803.294020:0:63851:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 264 ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00000002:00010000:0.0:1769074803.294034:0:63851:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074803.294050:0:63851:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03493af000 type 13 mode 16 res 8589935621/510 (0 0) 00000080:00200000:0.0:1769074803.294057:0:63851:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200000405:0x1fe:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1769074803.294073:0:63851:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x1fe:0x0](ffff9b0355ce1148) for lock 0xb7d6bf619879c473 00000080:00010000:0.0:1769074803.294080:0:63851:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c473 from it ffffb43641dc3bc8 00010000:00010000:0.0:1769074803.294085:0:63851:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074803.294094:0:63851:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074803.294117:0:63851:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074803.294127:0:63851:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00000080:00010000:0.0:1769074803.294134:0:63851:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000405:0x1fe:0x0](00000000ce6b6d44) being reconfigured ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074803.294177:0:63851:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074803.294185:0:63851:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074803.294199:0:63851:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fe:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1769074803.294212:0:63851:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074803.294280:0:63851:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fe:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:0.0:1769074803.294294:0:63851:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074803.294303:0:63851:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00000080:00010000:0.0:1769074803.294309:0:63851:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000405:0x1fe:0x0](00000000ce6b6d44) being reconfigured ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074803.294325:0:63851:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x1fe:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1769074803.294330:0:63851:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074803.294338:0:63851:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fe:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:0.0:1769074803.294352:0:63851:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 2097152 blocks 4096 [cma]time: 1769074798 1769074798 0 by oinfo size 2097152 blocks 4096 [cma]time 1769074803 1769074798 1769074798 ns: lustre-OST0000-osc-ffff9b035d92c800 lock: ffff9b0365861800/0xb7d6bf619879c46c lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd549a4 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074803.305222:0:63851:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job lfs:2274e3cf-6162-4197-bb1e-d97abc9b4bb9:63851:1855007616969600:192.168.204.160@tcp:35:lfs.0 00010000:00010000:0.0:1769074805.139388:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074806.410820:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625204608 00000100:00100000:0.0:1769074806.410831:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625204608 00000100:00100000:0.0:1769074806.410893:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8905 00000100:00100000:0.0:1769074806.410903:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:2274e3cf-6162-4197-bb1e-d97abc9b4bb9+5:6512:x1855007625204608:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074806.410918:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b035d92c800 lock: ffff9b0350989000/0xb7d6bf619879c473 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x1fe:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd549b9 expref: -99 pid: 63851 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074806.410992:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074806.410998:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce8a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:2274e3cf-6162-4197-bb1e-d97abc9b4bb9+5:6512:x1855007625204608:12345-192.168.204.160@tcp:104: Request processed in 98us (295us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074806.411009:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8905 00000100:00100000:0.0:1769074806.463283:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625205120 00000100:00100000:0.0:1769074806.463297:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625205120 00000100:00100000:0.0:1769074806.463361:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8906 00000100:00100000:0.0:1769074806.463371:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ceb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8404:x1855007625205120:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074806.463388:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b035d92c800 lock: ffff9b0365861800/0xb7d6bf619879c46c lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x420000020000 nid: local remote: 0x7fe2323adfd549a4 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074806.463467:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074806.463475:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ceb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8404:x1855007625205120:12345-192.168.204.160@tcp:104: Request processed in 106us (360us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074806.463482:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8906 00000080:00200000:0.0:1769074806.606974:0:63717:0:(file.c:2109:ll_file_io_generic()) Range unlock [256, 511] 00000080:00200000:0.0:1769074806.606988:0:63717:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x1fe:0x0] ignore/verify layout 0/0, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074806.606992:0:63717:0:(file.c:2142:ll_file_io_generic()) f200c.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:0.0:1769074806.607005:0:63717:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x1fe:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:0.0:1769074806.607008:0:63717:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074806.607016:0:63717:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:0.0:1769074806.607056:0:63717:0:(file.c:424:ll_file_release()) START file f200c.sanity-flr:[0x200000405:0x1fe:0x0](ffff9b0355ce21c8), flags 140001 00000002:00100000:0.0:1769074806.607124:0:63717:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b03482c4380 x1855007616962304/t8589944039(8589944039) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 800/872 e 1 to 0 dl 1769074841 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074806.607135:0:63717:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074806.607140:0:63717:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a39f80 x1855007616971008/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074806.607163:0:63717:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:63717:1855007616971008:192.168.204.160@tcp:35:dd.0 00000100:00100000:0.0:1769074806.607375:0:63717:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0342f69780 going to sleep for 11 seconds 00000100:00100000:0.0:1769074806.610459:0:63717:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:63717:1855007616971008:192.168.204.160@tcp:35:dd.0 00000002:00100000:0.0:1769074806.610498:0:63717:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b03482c4380 x1855007616962304/t8589944039(8589944039) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 800/872 e 1 to 0 dl 1769074841 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000001:02000400:0.0:1769074808.245349:0:64131:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: dd completed 00000100:00100000:0.0:1769074808.434009:0:64137:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64137:1855007616972928:192.168.204.160@tcp:34:lfs.0 00000080:00200008:0.0:1769074808.436577:0:64137:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 304000 00000020:00200000:0.0:1769074808.537079:0:64143:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000080:00200000:0.0:1769074808.537089:0:64143:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 2097152 count 2097152 file_offset 2097152 00000020:00010000:0.0:1769074808.537100:0:64143:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03439d37b8 (ffff9b0342ca8998/0) at cl_lock_release():245 00000020:00010000:0.0:1769074808.537103:0:64143:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03439d37b8 (ffff9b0342ca8998/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074808.537106:0:64143:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359274e30 (ffff9b0342ca8e18/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074808.537108:0:64143:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03439d37b8 (ffff9b0342ca8998/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074808.537110:0:64143:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359274e30 (ffff9b0342ca8998/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074808.537118:0:64143:0:(vvp_io.c:518:vvp_io_advance()) advancing 2097152 bytes 00000020:00200000:0.0:1769074808.537125:0:64143:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1769074808.537143:0:28211:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b03515b9900 00000008:00100000:0.0:1769074808.537351:0:28211:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a39f80 - object 0x280000401:706 offset 0<>2097152 00000100:00100000:0.0:1769074808.537400:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616973952:192.168.204.160@tcp:3:lfs.0 00000100:00100000:0.0:1769074808.543618:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625206912 00000100:00100000:0.0:1769074808.543633:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625206912 00000100:00100000:0.0:1769074808.543706:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8907 00000100:00100000:0.0:1769074808.543725:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8404:x1855007625206912:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074808.543742:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b035d92c800 lock: ffff9b0365861c00/0xb7d6bf619879c457 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd54981 expref: -99 pid: 63851 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074808.544757:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074808.544784:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8404:x1855007625206912:12345-192.168.204.160@tcp:104: Request processed in 1056us (2581us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074808.544794:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8907 00000080:00200080:0.0:1769074808.588402:0:64143:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x1fe:0x0](ffff9b0355ce21c8) cmd=c00c66f3 arg=7ffdf3d817d4 00010000:00010000:0.0:1769074808.588428:0:64143:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c4b2 lrc: 2/0,0 mode: PW/PW res: [0x200000405:0x1fe:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7fe2323adfd54aa0 expref: -99 pid: 64143 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074808.588487:0:64143:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c4b2 lrc: 1/0,0 mode: --/PW res: [0x200000405:0x1fe:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x85809400000020 nid: local remote: 0x7fe2323adfd54aa0 expref: -99 pid: 64143 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074808.588506:0:64143:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:0.0:1769074808.588530:0:64143:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64143:1855007616974208:192.168.204.160@tcp:103:lfs.0 00000100:00100000:0.0:1769074808.588608:0:64143:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0348cd7000 going to sleep for 11 seconds 00010000:00010000:0.0:1769074812.307608:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074817.586611:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074817.619455:0:39330:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074817.619474:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625212032 00000100:00100000:0.0:1769074817.619481:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625212032 00000100:00100000:0.0:1769074817.619527:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8908 00000100:00100000:0.0:1769074817.619537:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0350461c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:27843:x1855007625212032:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074817.619545:0:39330:0:(ldlm_lockd.c:2447:ldlm_callback_handler()) callback on lock 0xb7d6bf619879c45e - lock disappeared 00010000:00010000:0.0:1769074817.619596:0:39330:0:(ldlm_lockd.c:2344:ldlm_callback_errmsg()) @@@ Operate with invalid parameter, NID=12345-192.168.204.160@tcp lock=0xb7d6bf619879c45e: rc = 0 req@ffff9b0350461c50 x1855007625212032/t0(0) o104->LOV_OSC_UUID@192.168.204.160@tcp:353/0 lens 328/224 e 0 to 0 dl 1769074828 ref 1 fl Interpret:/0/0 rc -22/-22 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074817.619607:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0350461c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:27843:x1855007625212032:12345-192.168.204.160@tcp:104: Request processed in 73us (307us total) trans 0 rc -22/-22 00000100:00100000:0.0:1769074817.619613:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8908 00010000:00010000:0.0:1769074817.619900:0:60220:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074817.619907:0:60220:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c4a4 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd54a7d expref: -99 pid: 64143 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074817.619916:0:60220:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0342cd0800 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:0.0:1769074817.619922:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c4a4 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x428400010000 nid: local remote: 0x7fe2323adfd54a7d expref: -99 pid: 64143 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074817.619946:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c4a4 lrc: 2/0,0 mode: --/PR res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd54a7d expref: -99 pid: 64143 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074817.619953:0:60220:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074817.619968:0:60220:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c4a4 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd54a7d expref: -99 pid: 64143 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074817.619975:0:60220:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c4a4 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2c1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd54a7d expref: -99 pid: 64143 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074817.619991:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074817.619992:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074817.619994:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074817.620005:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616977152:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074817.631548:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616977152:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00020000:0.0:1769074817.648821:0:64591:0:(obd_class.h:479:obd_check_dev()) Device 12 not setup 00000020:00000080:0.0:1769074817.649188:0:64591:0:(obd_config.c:1389:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1769074817.649203:0:64591:0:(obd_config.c:814:class_detach()) detach on obd lustre-OST0001-osc-ffff9b035d92c800 (uuid 2274e3cf-6162-4197-bb1e-d97abc9b4bb9) 00020000:01000000:0.0:1769074817.649208:0:64591:0:(lov_obd.c:611:__lov_del_obd()) Removing tgt lustre-OST0000_UUID : lustre-OST0000-osc-ffff9b035d92c800 00020000:01000000:0.0:1769074817.649211:0:64591:0:(lov_obd.c:248:lov_disconnect_obd()) lustre-clilov-ffff9b035d92c800: disconnecting target lustre-OST0000-osc-ffff9b035d92c800 00000100:00080000:0.0:1769074817.649219:0:64591:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 2274e3cf-6162-4197-bb1e-d97abc9b4bb9->lustre-OST0000_UUID 00000100:00080000:0.0:1769074817.649395:0:64591:0:(import.c:65:import_set_state_nolock()) ffff9b03593e0800 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:00080000:0.0:1769074817.657733:0:64591:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00100000:0.0:1769074817.657747:0:64591:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9b035d92c800: committing for last_committed 21474841276 gen 2 00000100:00100000:0.0:1769074817.657762:0:64591:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:0.0:1769074817.657916:0:64591:0:(genops.c:1485:class_disconnect()) disconnect: cookie 0xb7d6bf619879c3cb 00000020:00000080:0.0:1769074817.657943:0:64591:0:(genops.c:904:class_export_put()) final put ffff9b0372745800/LOV_OSC_UUID 00000020:01000000:0.0:1769074817.658502:0:64591:0:(obd_config.c:2361:class_manual_cleanup()) Manual cleanup of lustre-OST0000-osc-ffff9b035d92c800 (flags='') 00000020:00000080:0.0:1769074817.658509:0:64591:0:(obd_config.c:1389:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1769074817.658523:0:64591:0:(obd_config.c:886:class_cleanup()) lustre-OST0000-osc-ffff9b035d92c800: forcing exports to disconnect: 0/3 00000020:00080000:0.0:1769074817.658527:0:64591:0:(genops.c:1575:class_disconnect_exports()) OBD device 11 (ffff9b0342f131b0) has no exports 00000020:00000080:0.0:1769074817.691056:0:64591:0:(genops.c:1485:class_disconnect()) disconnect: cookie 0xb7d6bf619879c3af 00000020:00000080:0.0:1769074817.691065:0:64591:0:(genops.c:904:class_export_put()) final put ffff9b035d92d000/2274e3cf-6162-4197-bb1e-d97abc9b4bb9 00000020:01000000:0.0:1769074817.691184:0:64591:0:(obd_config.c:2361:class_manual_cleanup()) Manual cleanup of lustre-clilov-ffff9b035d92c800 (flags='') 00000020:00000080:0.0:1769074817.691195:0:64591:0:(obd_config.c:1389:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1769074817.691205:0:64591:0:(obd_config.c:886:class_cleanup()) lustre-clilov-ffff9b035d92c800: forcing exports to disconnect: 0/1 00000020:00080000:0.0:1769074817.691210:0:64591:0:(genops.c:1575:class_disconnect_exports()) OBD device 7 (ffff9b035080e360) has no exports 00000020:00000080:0.0:1769074817.691279:0:64591:0:(obd_config.c:1389:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1769074817.691288:0:64591:0:(obd_config.c:814:class_detach()) detach on obd lustre-clilov-ffff9b035d92c800 (uuid 2274e3cf-6162-4197-bb1e-d97abc9b4bb9) 00000020:00000080:0.0:1769074817.691291:0:64591:0:(genops.c:904:class_export_put()) final put ffff9b0352a94800/2274e3cf-6162-4197-bb1e-d97abc9b4bb9 00000020:00000080:0.0:1769074817.691294:0:64591:0:(genops.c:863:class_export_destroy()) destroying export ffff9b0352a94800/2274e3cf-6162-4197-bb1e-d97abc9b4bb9 for lustre-clilov-ffff9b035d92c800 00000020:01000000:0.0:1769074817.691307:0:64591:0:(genops.c:443:class_free_dev()) finishing cleanup of obd lustre-clilov-ffff9b035d92c800 (2274e3cf-6162-4197-bb1e-d97abc9b4bb9) 00000020:01000000:0.0:1769074817.691319:0:64591:0:(obd_config.c:2361:class_manual_cleanup()) Manual cleanup of lustre-clilmv-ffff9b035d92c800 (flags='') 00000020:00000080:0.0:1769074817.691323:0:64591:0:(obd_config.c:1389:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1769074817.691326:0:64591:0:(obd_config.c:886:class_cleanup()) lustre-clilmv-ffff9b035d92c800: forcing exports to disconnect: 0/2 00000020:00080000:0.0:1769074817.691328:0:64591:0:(genops.c:1575:class_disconnect_exports()) OBD device 8 (ffff9b035080b1b0) has no exports 00000020:00000080:0.0:1769074817.691410:0:64591:0:(obd_config.c:1389:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1769074817.691414:0:64591:0:(obd_config.c:814:class_detach()) detach on obd lustre-clilmv-ffff9b035d92c800 (uuid 2274e3cf-6162-4197-bb1e-d97abc9b4bb9) 00000020:01000000:0.0:1769074817.691418:0:64591:0:(obd_config.c:2361:class_manual_cleanup()) Manual cleanup of lustre-MDT0000-mdc-ffff9b035d92c800 (flags='') 00000020:00000080:0.0:1769074817.691420:0:64591:0:(obd_config.c:1389:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1769074817.691424:0:64591:0:(obd_config.c:886:class_cleanup()) lustre-MDT0000-mdc-ffff9b035d92c800: forcing exports to disconnect: 0/2 00000020:00080000:0.0:1769074817.691426:0:64591:0:(genops.c:1575:class_disconnect_exports()) OBD device 9 (ffff9b035080ca88) has no exports 00000008:01000000:0.0:1769074817.692594:0:64591:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-MDT0000-mdc-ffff9b035d92c800: client import never connected 00000100:00080000:0.0:1769074817.692601:0:64591:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00100000:0.0:1769074817.692607:0:64591:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b035d92c800: committing for last_committed 8589944038 gen 3 00000100:00020000:0.0:1769074817.692614:0:64591:0:(obd_class.h:479:obd_check_dev()) Device 9 not setup 00000100:00100000:0.0:1769074817.692617:0:64591:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00020000:0.0:1769074817.692620:0:64591:0:(obd_class.h:479:obd_check_dev()) Device 9 not setup 00000020:00000080:0.0:1769074817.692960:0:64591:0:(obd_config.c:1389:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1769074817.692970:0:64591:0:(obd_config.c:814:class_detach()) detach on obd lustre-MDT0000-mdc-ffff9b035d92c800 (uuid 2274e3cf-6162-4197-bb1e-d97abc9b4bb9) 00000020:01000000:0.0:1769074817.692975:0:64591:0:(obd_config.c:2361:class_manual_cleanup()) Manual cleanup of lustre-MDT0001-mdc-ffff9b035d92c800 (flags='') 00000020:00000080:0.0:1769074817.692981:0:64591:0:(obd_config.c:1389:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1769074817.692985:0:64591:0:(obd_config.c:886:class_cleanup()) lustre-MDT0001-mdc-ffff9b035d92c800: forcing exports to disconnect: 0/2 00000020:00080000:0.0:1769074817.692989:0:64591:0:(genops.c:1575:class_disconnect_exports()) OBD device 10 (ffff9b0352a718d8) has no exports 00000008:01000000:0.0:1769074817.695087:0:64591:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-MDT0001-mdc-ffff9b035d92c800: client import never connected 00000100:00080000:0.0:1769074817.695098:0:64591:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0001_UUID INVALID 00000100:00100000:0.0:1769074817.695103:0:64591:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9b035d92c800: committing for last_committed 0 gen 3 00000100:00020000:0.0:1769074817.695111:0:64591:0:(obd_class.h:479:obd_check_dev()) Device 10 not setup 00000100:00100000:0.0:1769074817.695114:0:64591:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00020000:0.0:1769074817.695117:0:64591:0:(obd_class.h:479:obd_check_dev()) Device 10 not setup 00000020:00000080:0.0:1769074817.695465:0:64591:0:(obd_config.c:1389:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1769074817.695490:0:64591:0:(obd_config.c:814:class_detach()) detach on obd lustre-MDT0001-mdc-ffff9b035d92c800 (uuid 2274e3cf-6162-4197-bb1e-d97abc9b4bb9) 00000080:02000400:0.0:1769074817.695497:0:64591:0:(llite_lib.c:1613:ll_put_super()) Unmounted lustre-client 00000020:01000000:0.0:1769074817.699418:0:64591:0:(obd_config.c:1123:class_del_profile()) Del profile lustre-client 00000020:01000004:0.0:1769074817.699634:0:64591:0:(obd_mount.c:875:lustre_common_put_super()) dropping sb ffff9b035d92c800 00000020:01000004:0.0:1769074817.699641:0:64591:0:(obd_mount.c:569:lustre_stop_mgc()) MGC still has 1 references. 00000020:01000004:0.0:1769074817.699644:0:64591:0:(obd_mount.c:888:lustre_common_put_super()) MGC still in use 00000020:01000004:0.0:1769074817.699647:0:64591:0:(obd_mount.c:705:lustre_put_lsi()) put ffff9b035d92c800 1 00000020:01000004:0.0:1769074817.699650:0:64591:0:(obd_mount.c:637:lustre_free_lsi()) Freeing lsi ffff9b035d92f800 00000100:00100000:0.0:1769074821.946823:0:64878:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64878:1855007616978688:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074821.946857:0:64878:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074821.946898:0:64878:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 4/0,1 mode: --/PR res: [0x200000405:0x200:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074821.946906:0:64878:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 4/0,1 mode: --/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074821.946919:0:64878:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 4/0,1 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074821.946930:0:64878:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 4/0,1 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074821.946943:0:64878:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 5/1,1 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074821.946950:0:64878:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 5/1,1 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074821.946957:0:64878:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074821.946969:0:64878:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=8000037, status=0 req@ffff9b0352a1b100 x1855007616978688/t8589944048(8589944048) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 528/608 e 0 to 0 dl 1769074876 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1769074821.946982:0:64878:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0352a1b100 x1855007616978688/t8589944048(8589944048) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 528/608 e 0 to 0 dl 1769074876 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074821.946995:0:64878:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074821.947010:0:64878:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/512 (0 0) 00000080:00200000:0.0:1769074821.947017:0:64878:0:(namei.c:955:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1769074821.947032:0:64878:0:(namei.c:1569:ll_create_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=open|creat 00000080:00200000:0.0:1769074821.947072:0:64878:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200000405:0x200:0x0](ffff9b03606fd348) 00000080:00200000:0.0:1769074821.947081:0:64878:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x200:0x0], updating i_size 0 00000080:00200000:0.0:1769074821.947170:0:64878:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x200:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1769074821.947176:0:64878:0:(namei.c:1535:ll_create_node()) setting l_ast_data to inode [0x200000405:0x200:0x0](ffff9b03606fd348) 00000080:00010000:0.0:1769074821.947179:0:64878:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x200:0x0](ffff9b03606fd348) for lock 0xb7d6bf619879c4ce 00000080:00010000:0.0:1769074821.947191:0:64878:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x200:0x0](ffff9b03606fd348) 00000080:00200008:0.0:1769074821.947207:0:64878:0:(file.c:1001:ll_file_open()) START file f202.sanity-flr:[0x200000405:0x200:0x0](ffff9b03606fd348), flags 120501 00000002:00100000:0.0:1769074821.947264:0:64878:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0352a1b100 x1855007616978688/t8589944048(8589944048) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 528/608 e 0 to 0 dl 1769074876 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1769074821.947309:0:64878:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c4ce from it ffff9b03429f4180 00010000:00010000:0.0:1769074821.947314:0:64878:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074821.947323:0:64878:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c4ce lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54b25 expref: -99 pid: 64878 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074821.947347:0:64878:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x200:0x0](ffff9b03606fd348), inode mode 81a4 mask 40 00000080:00200000:0.0:1769074821.947411:0:64878:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x200:0x0](ffff9b03606fd348), inode mode 81a4 mask 2 00000080:00200000:0.0:1769074821.947422:0:64878:0:(xattr.c:406:ll_xattr_set()) VFS Op:inode=[0x200000405:0x200:0x0](ffff9b03606fd348), xattr lov 00000002:00010000:0.0:1769074821.947443:0:64878:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x200:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 040000002 00010000:00010000:0.0:1769074821.947507:0:64878:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c4d5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64878 initiator: MDT0 00010000:00010000:0.0:1769074821.947515:0:64878:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c4d5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64878 initiator: MDT0 00000020:00100000:0.0:1769074821.947527:0:64878:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074821.947533:0:64878:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a39180 x1855007616978944/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074821.947542:0:64878:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c4d5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64878 initiator: MDT0 00000100:00100000:0.0:1769074821.947567:0:64878:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64878:1855007616978944:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074821.947781:0:64878:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2f00 going to sleep for 50 seconds 00000100:00100000:0.0:1769074821.999703:0:64878:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64878:1855007616979328:192.168.204.160@tcp:35:lfs.0 00000002:00100000:0.0:1769074821.999749:0:64878:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0365a39180 x1855007616978944/t8589944049(8589944049) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 656/744 e 0 to 0 dl 1769074876 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:0.0:1769074821.999807:0:64878:0:(file.c:424:ll_file_release()) START file f202.sanity-flr:[0x200000405:0x200:0x0](ffff9b03606fd348), flags 100001 00000002:00100000:0.0:1769074821.999870:0:64878:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0352a1b100 x1855007616978688/t8589944048(8589944048) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 528/608 e 0 to 0 dl 1769074876 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074821.999881:0:64878:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074821.999887:0:64878:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a39500 x1855007616979584/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074821.999912:0:64878:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64878:1855007616979584:192.168.204.160@tcp:35:lfs.0 00000100:00100000:0.0:1769074822.000024:0:64878:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2900 going to sleep for 11 seconds 00000100:00100000:0.0:1769074822.003486:0:64878:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64878:1855007616979584:192.168.204.160@tcp:35:lfs.0 00000002:00100000:0.0:1769074822.003531:0:64878:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0352a1b100 x1855007616978688/t8589944048(8589944048) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 528/608 e 0 to 0 dl 1769074876 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:0.0:1769074822.242466:0:57730:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074822.242482:0:57730:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c4ff lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7fe2323adfd54bd4 expref: -99 pid: 64894 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.242497:0:57730:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365861400 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074822.242504:0:57730:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c4ff lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7fe2323adfd54bd4 expref: -99 pid: 64894 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.242514:0:57730:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c4ff lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7fe2323adfd54bd4 expref: -99 pid: 64894 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.242525:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c4ff lrc: 3/0,0 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd54bd4 expref: -99 pid: 64894 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.242554:0:57730:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c4f1 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54bbf expref: -99 pid: 64894 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.242615:0:57730:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c4ff lrc: 2/0,0 mode: --/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd54bd4 expref: -99 pid: 64894 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.242623:0:57730:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074822.242662:0:57730:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c4ff lrc: 1/0,0 mode: --/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd54bd4 expref: -99 pid: 64894 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.242670:0:57730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c4ff lrc: 0/0,0 mode: --/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd54bd4 expref: -99 pid: 64894 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.242691:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074822.242693:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074822.242695:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074822.265119:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625213312 00000100:00100000:0.0:1769074822.265133:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625213312 00000100:00100000:0.0:1769074822.265200:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8910 00000100:00100000:0.0:1769074822.265210:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ceb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6514:x1855007625213312:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074822.265228:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c4f1 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x200:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd54bbf expref: -99 pid: 64894 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.265368:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074822.265375:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ceb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6514:x1855007625213312:12345-192.168.204.160@tcp:104: Request processed in 167us (384us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074822.265382:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8910 00000100:00100000:0.0:1769074822.285497:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074822.323857:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000080:00200000:0.0:1769074822.562618:0:64902:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074822.562663:0:64902:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074822.562686:0:64902:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c4dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b4f expref: -99 pid: 64885 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074822.562707:0:64902:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c4dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b4f expref: -99 pid: 64885 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074822.562716:0:64902:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074822.562726:0:64902:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074822.562731:0:64902:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c4dc 00000080:00010000:0.0:1769074822.562740:0:64902:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c4dc from it ffffb43640d7fbf8 00010000:00010000:0.0:1769074822.562745:0:64902:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c4dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54b4f expref: -99 pid: 64885 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074822.562756:0:64902:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c4dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b4f expref: -99 pid: 64885 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074822.562779:0:64902:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074822.562799:0:64902:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), file ffff9b0360858480, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1769074822.562807:0:64902:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f202.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=open 00000002:00010000:0.0:1769074822.562840:0:64902:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f202.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:0.0:1769074822.562912:0:64902:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 64902 initiator: MDT0 00010000:00010000:0.0:1769074822.562920:0:64902:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64902 initiator: MDT0 00000020:00100000:0.0:1769074822.562936:0:64902:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074822.562952:0:64902:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3a300 x1855007616983680/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2160/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074822.562963:0:64902:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64902 initiator: MDT0 00000100:00100000:0.0:1769074822.562989:0:64902:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64902:1855007616983680:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074822.563794:0:64902:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2b40 going to sleep for 50 seconds 00000100:00100000:0.0:1769074822.574444:0:64902:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64902:1855007616983680:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074822.574477:0:64902:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 64902 initiator: MDT0 00010000:00010000:0.0:1769074822.574493:0:64902:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64902 initiator: MDT0 00010000:00010000:0.0:1769074822.574502:0:64902:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64902 initiator: MDT0 00010000:00010000:0.0:1769074822.574515:0:64902:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 64902 initiator: MDT0 00010000:00010000:0.0:1769074822.574524:0:64902:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 64902 initiator: MDT0 00010000:00010000:0.0:1769074822.574531:0:64902:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0349292200 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074822.574537:0:64902:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64902 initiator: MDT0 00010000:00010000:0.0:1769074822.574545:0:64902:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 64902 initiator: MDT0 00010000:00010000:0.0:1769074822.574554:0:64902:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 64902 initiator: MDT0 00010000:00010000:0.0:1769074822.574561:0:64902:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074822.574570:0:64902:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64902 initiator: MDT0 00010000:00010000:0.0:1769074822.574576:0:64902:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c522 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 64902 initiator: MDT0 00000002:00100000:0.0:1769074822.574613:0:64902:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0365a3a300 x1855007616983680/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2160/608 e 0 to 0 dl 1769074877 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074822.574628:0:64902:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1769074822.574821:0:64902:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074822.574848:0:64902:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074822.574864:0:64902:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c4dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b4f expref: -99 pid: 64885 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074822.574883:0:64902:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c4dc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b4f expref: -99 pid: 64885 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074822.574892:0:64902:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074822.574898:0:64902:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074822.574902:0:64902:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c4dc 00000080:00010000:0.0:1769074822.574908:0:64902:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c4dc from it ffffb43640d7fbb0 00010000:00010000:0.0:1769074822.574912:0:64902:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c4dc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54b4f expref: -99 pid: 64885 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074822.574920:0:64902:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c4dc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54b4f expref: -99 pid: 64885 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074822.574936:0:64902:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074822.574950:0:64902:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 304000 00000080:00200080:0.0:1769074822.574992:0:64902:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=c0086915 arg=16503d0 00000100:00100000:0.0:1769074822.575067:0:64902:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64902:1855007616983808:192.168.204.160@tcp:34:lfs.0 00000100:00100000:0.0:1769074822.575198:0:64902:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2b40 going to sleep for 50 seconds 00000100:00100000:0.0:1769074822.579419:0:64902:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64902:1855007616983808:192.168.204.160@tcp:34:lfs.0 00000100:00100000:0.0:1769074822.677629:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074822.677681:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625213696 00000100:00100000:0.0:1769074822.677691:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625213696 00000100:00100000:0.0:1769074822.677746:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8913 00000100:00100000:0.0:1769074822.677756:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a3a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:27843:x1855007625213696:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074822.677773:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860c00/0xb7d6bf619879c4f8 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd54bc6 expref: -99 pid: 64894 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.677851:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074822.677858:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a3a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:27843:x1855007625213696:12345-192.168.204.160@tcp:104: Request processed in 104us (241us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074822.677865:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8913 00000100:00100000:0.0:1769074822.689936:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074822.689971:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625213824 00000100:00100000:0.0:1769074822.689979:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625213824 00000100:00100000:0.0:1769074822.690025:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8914 00000100:00100000:0.0:1769074822.690033:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a3b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8424:x1855007625213824:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074822.690042:0:39330:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 5/0,1 mode: --/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x80020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.690051:0:39330:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 5/0,1 mode: --/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x80020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.690058:0:39330:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 5/0,1 mode: --/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000080020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.690067:0:39330:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080080020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.690075:0:39330:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080080020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.690085:0:39330:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x80080020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.690127:0:39330:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b0342cd1a00) 00000100:00100000:0.0:1769074822.701474:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a3b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8424:x1855007625213824:12345-192.168.204.160@tcp:105: Request processed in 11429us (11526us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074822.701490:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8914 00000100:00100000:0.0:1769074822.787995:0:64903:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64903:1855007616985216:192.168.204.160@tcp:101:dd.0 00010000:00010000:0.0:1769074822.788022:0:64903:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 4/0,1 mode: --/EX res: [0x200000405:0x200:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788035:0:64903:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 4/0,1 mode: --/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788046:0:64903:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 4/0,1 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788057:0:64903:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 4/0,1 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788080:0:64903:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 5/1,1 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788090:0:64903:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 5/1,1 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788097:0:64903:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00000002:00100000:0.0:1769074822.788107:0:64903:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9b0352a1a300 x1855007616985216/t8589944061(8589944061) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 376/608 e 0 to 0 dl 1769074877 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074822.788126:0:64903:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 264 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00000002:00010000:0.0:1769074822.788138:0:64903:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788160:0:64903:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 16 res 8589935621/512 (0 0) 00000080:00200000:0.0:1769074822.788167:0:64903:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200000405:0x200:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1769074822.788177:0:64903:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x200:0x0](ffff9b03606fd348) for lock 0xb7d6bf619879c545 00000080:00010000:0.0:1769074822.788185:0:64903:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c545 from it ffffb43640e33bb8 00010000:00010000:0.0:1769074822.788188:0:64903:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788196:0:64903:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074822.788211:0:64903:0:(file.c:2142:ll_file_io_generic()) f202.sanity-flr: 2 io complete with rc: 0, result: 0, restart: 1 00000080:00200000:0.0:1769074822.788214:0:64903:0:(file.c:2149:ll_file_io_generic()) f202.sanity-flr: restart write from ppos=1048576 bytes=1048576 retries=999 ret=0: rc = 0 00000080:00200000:0.0:1769074822.788220:0:64903:0:(file.c:1759:ll_io_set_mirror()) f202.sanity-flr: desiginated mirror: 0 00000020:00200000:0.0:1769074822.788229:0:64903:0:(cl_io.c:214:cl_io_rw_init()) header@00000000f776d590[0x4, 257, [0x200000405:0x200:0x0] hash] 00000020:00200000:0.0:1769074822.788231:0:64903:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [1048576, 2097152) 0 0 00000080:00200000:0.0:1769074822.788265:0:64903:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x200:0x0] ignore/verify layout 0/0, layout version 2 restore needed 0 00010000:00010000:0.0:1769074822.788283:0:64903:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788293:0:64903:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00000080:00010000:0.0:1769074822.788305:0:64903:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000405:0x200:0x0](000000003b8f47b6) being reconfigured ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788340:0:64903:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074822.788347:0:64903:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0600/0xb7d6bf619879c545 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x200:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cd7 expref: -99 pid: 64903 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074822.788386:0:64903:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1769074822.788395:0:64903:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1769074822.788417:0:64903:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000008:00010000:0.0:1769074822.788433:0:64903:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### extent: 00000000e7986e66 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020080020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074822.788447:0:64903:0:(osc_cache.c:1103:osc_extent_make_ready()) ### extent: 00000000e7986e66 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020080020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.788973:0:64903:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020080020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00100000:0.0:1769074822.789917:0:64903:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b0365a3bb80 - object 0x280000401:707 offset 0<>1048576 00000100:00100000:0.0:1769074822.948619:0:2406:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9b0342ad2800: committing for last_committed 21474841283 gen 1 00000100:00100000:0.0:1769074822.948651:0:2406:0:(client.c:3125:ptlrpc_free_committed()) @@@ stopping search req@ffff9b0365a3bb80 x1855007616985344/t21474841285(21474841285) o4->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:6/4 lens 488/448 e 0 to 0 dl 1769074838 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1769074822.948719:0:2406:0:(osc_cache.c:861:osc_extent_finish()) ### extent: 00000000e7986e66 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020080020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074822.948942:0:2406:0:(osc_cache.c:483:__osc_extent_remove()) ### extent: 00000000e7986e66 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020080020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074822.948971:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616985344:192.168.204.160@tcp:4:dd.0 00000100:00100000:0.0:1769074822.954330:0:64903:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360be06c0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074822.956862:0:64903:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64903:1855007616985600:192.168.204.160@tcp:101:dd.0 00010000:00010000:0.0:1769074822.956907:0:64903:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.956927:0:64903:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.956941:0:64903:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.956956:0:64903:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074822.956969:0:64903:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,2 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.956982:0:64903:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074822.956989:0:64903:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074822.957015:0:64903:0:(vvp_io.c:1312:vvp_io_write_start()) f202.sanity-flr: write [1048576, 2097152) append 0 append lockpos 0 00000080:00200000:0.0:1769074822.957035:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 256 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957103:0:64903:0:(rw26.c:912:ll_write_end()) pos 1048576, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957114:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000c3fe1e47[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957117:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000a827e8fc 17ffffc0001001 3:0 ffff9b0352bffaf8 256 lru 00000080:00200000:0.0:1769074822.957125:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000bdff8599 256: 1< 0 - - > 2< 1048576 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957127:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000c3fe1e47 00000080:00200000:0.0:1769074822.957128:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 1. 00000080:00200000:0.0:1769074822.957135:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 257 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957162:0:64903:0:(rw26.c:912:ll_write_end()) pos 1052672, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957166:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a9fe2f02[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957169:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000025083301 17ffffc0001001 3:0 ffff9b0352bff798 257 lru 00000080:00200000:0.0:1769074822.957175:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e7029778 257: 1< 0 - - > 2< 1052672 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957177:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a9fe2f02 00000080:00200000:0.0:1769074822.957178:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 2. 00000080:00200000:0.0:1769074822.957180:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 258 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957197:0:64903:0:(rw26.c:912:ll_write_end()) pos 1056768, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957201:0:64903:0:(rw26.c:969:ll_write_end()) page@000000006d0bf498[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957204:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000dcc551be 17ffffc0001001 3:0 ffff9b0352bff1b0 258 lru 00000080:00200000:0.0:1769074822.957210:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006c9a5fea 258: 1< 0 - - > 2< 1056768 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957212:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000006d0bf498 00000080:00200000:0.0:1769074822.957213:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 3. 00000080:00200000:0.0:1769074822.957216:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 259 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957230:0:64903:0:(rw26.c:912:ll_write_end()) pos 1060864, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957281:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000288e471a[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957284:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000004f0edf92 17ffffc0001001 3:0 ffff9b0352bffa20 259 lru 00000080:00200000:0.0:1769074822.957290:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000000b852f3a 259: 1< 0 - - > 2< 1060864 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957292:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000288e471a 00000080:00200000:0.0:1769074822.957294:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 4. 00000080:00200000:0.0:1769074822.957297:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 260 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957315:0:64903:0:(rw26.c:912:ll_write_end()) pos 1064960, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957319:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000e9153297[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957322:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000dae484c6 17ffffc0001001 3:0 ffff9b0352bff360 260 lru 00000080:00200000:0.0:1769074822.957327:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000002933c115 260: 1< 0 - - > 2< 1064960 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957330:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000e9153297 00000080:00200000:0.0:1769074822.957331:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 5. 00000080:00200000:0.0:1769074822.957333:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 261 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957347:0:64903:0:(rw26.c:912:ll_write_end()) pos 1069056, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957351:0:64903:0:(rw26.c:969:ll_write_end()) page@000000009252db69[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957354:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000024d4cbca 17ffffc0001001 3:0 ffff9b0352bff6c0 261 lru 00000080:00200000:0.0:1769074822.957360:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000004b7be2e9 261: 1< 0 - - > 2< 1069056 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957363:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000009252db69 00000080:00200000:0.0:1769074822.957364:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 6. 00000080:00200000:0.0:1769074822.957366:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 262 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957378:0:64903:0:(rw26.c:912:ll_write_end()) pos 1073152, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957381:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000261cdc13[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957384:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000daf5492a 17ffffc0001001 3:0 ffff9b0352bffbd0 262 lru 00000080:00200000:0.0:1769074822.957389:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b9075462 262: 1< 0 - - > 2< 1073152 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957391:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000261cdc13 00000080:00200000:0.0:1769074822.957392:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 7. 00000080:00200000:0.0:1769074822.957394:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 263 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957406:0:64903:0:(rw26.c:912:ll_write_end()) pos 1077248, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957410:0:64903:0:(rw26.c:969:ll_write_end()) page@000000003a48ad01[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957412:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000000f425955 17ffffc0001001 3:0 ffff9b0352bff870 263 lru 00000080:00200000:0.0:1769074822.957418:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e656c77d 263: 1< 0 - - > 2< 1077248 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957420:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000003a48ad01 00000080:00200000:0.0:1769074822.957421:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 8. 00000080:00200000:0.0:1769074822.957423:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 264 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957434:0:64903:0:(rw26.c:912:ll_write_end()) pos 1081344, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957437:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000013be3d8d[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957440:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000007c9afac8 17ffffc0001001 3:0 ffff9b0352bff288 264 lru 00000080:00200000:0.0:1769074822.957446:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000005f2579a6 264: 1< 0 - - > 2< 1081344 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957448:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000013be3d8d 00000080:00200000:0.0:1769074822.957449:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 9. 00000080:00200000:0.0:1769074822.957451:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 265 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957462:0:64903:0:(rw26.c:912:ll_write_end()) pos 1085440, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957466:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000055ec7ff2[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957469:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000004f690283 17ffffc0001001 3:0 ffff9b0352bff0d8 265 lru 00000080:00200000:0.0:1769074822.957474:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006daa7654 265: 1< 0 - - > 2< 1085440 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957476:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000055ec7ff2 00000080:00200000:0.0:1769074822.957477:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 10. 00000080:00200000:0.0:1769074822.957479:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 266 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957493:0:64903:0:(rw26.c:912:ll_write_end()) pos 1089536, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957497:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000ffa73417[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957500:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000000d35835f 17ffffc0001001 3:0 ffff9b0352bffd80 266 lru 00000080:00200000:0.0:1769074822.957506:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000df0a3f24 266: 1< 0 - - > 2< 1089536 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957508:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000ffa73417 00000080:00200000:0.0:1769074822.957509:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 11. 00000080:00200000:0.0:1769074822.957511:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 267 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957521:0:64903:0:(rw26.c:912:ll_write_end()) pos 1093632, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957525:0:64903:0:(rw26.c:969:ll_write_end()) page@000000005923a695[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957528:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000d7a76eee 17ffffc0001001 3:0 ffff9b0352bffe58 267 lru 00000080:00200000:0.0:1769074822.957533:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e6d2388d 267: 1< 0 - - > 2< 1093632 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957535:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000005923a695 00000080:00200000:0.0:1769074822.957536:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 12. 00000080:00200000:0.0:1769074822.957538:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 268 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957550:0:64903:0:(rw26.c:912:ll_write_end()) pos 1097728, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957554:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000e3a971ac[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957557:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000462056a7 17ffffc0001001 3:0 ffff9b0352bff438 268 lru 00000080:00200000:0.0:1769074822.957562:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000002dba0081 268: 1< 0 - - > 2< 1097728 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957564:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000e3a971ac 00000080:00200000:0.0:1769074822.957565:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 13. 00000080:00200000:0.0:1769074822.957567:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 269 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957578:0:64903:0:(rw26.c:912:ll_write_end()) pos 1101824, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957582:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000f2e6ad56[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957585:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000d1ce9055 17ffffc0001001 3:0 ffff9b0352bff510 269 lru 00000080:00200000:0.0:1769074822.957590:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000004bd1c410 269: 1< 0 - - > 2< 1101824 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957592:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000f2e6ad56 00000080:00200000:0.0:1769074822.957593:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 14. 00000080:00200000:0.0:1769074822.957595:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 270 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957613:0:64903:0:(rw26.c:912:ll_write_end()) pos 1105920, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957618:0:64903:0:(rw26.c:969:ll_write_end()) page@000000002efc5ee9[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957621:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000cce21611 17ffffc0001021 2:0 ffff9b0352bff948 270 lru 00000080:00200000:0.0:1769074822.957627:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000a216982f 270: 1< 0 - - > 2< 1105920 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957629:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000002efc5ee9 00000080:00200000:0.0:1769074822.957630:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 15. 00000080:00200000:0.0:1769074822.957631:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 271 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957644:0:64903:0:(rw26.c:912:ll_write_end()) pos 1110016, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957648:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000b608fd9c[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957651:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000c37d6da5 17ffffc0001001 3:0 ffff9b0352bff000 271 lru 00000080:00200000:0.0:1769074822.957656:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b424c79b 271: 1< 0 - - > 2< 1110016 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957659:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000b608fd9c 00000080:00200000:0.0:1769074822.957660:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 16. 00000080:00200000:0.0:1769074822.957661:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 272 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957674:0:64903:0:(rw26.c:912:ll_write_end()) pos 1114112, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957678:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000051619716[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957681:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000adb60e82 17ffffc0001001 3:0 ffff9b0352bffca8 272 lru 00000080:00200000:0.0:1769074822.957686:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000f5e45a5d 272: 1< 0 - - > 2< 1114112 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957689:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000051619716 00000080:00200000:0.0:1769074822.957690:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 17. 00000080:00200000:0.0:1769074822.957692:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 273 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957705:0:64903:0:(rw26.c:912:ll_write_end()) pos 1118208, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957709:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000024a3f32[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957712:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ee7a66bf 17ffffc0001001 3:0 ffff9b0352bff5e8 273 lru 00000080:00200000:0.0:1769074822.957718:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000043017405 273: 1< 0 - - > 2< 1118208 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957720:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000024a3f32 00000080:00200000:0.0:1769074822.957721:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 18. 00000080:00200000:0.0:1769074822.957723:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 274 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957738:0:64903:0:(rw26.c:912:ll_write_end()) pos 1122304, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957742:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000bd5d698a[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957745:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000008c42fed5 17ffffc0001001 3:0 ffff9b0342ae11b0 274 lru 00000080:00200000:0.0:1769074822.957750:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000f3b0292d 274: 1< 0 - - > 2< 1122304 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957752:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000bd5d698a 00000080:00200000:0.0:1769074822.957753:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 19. 00000080:00200000:0.0:1769074822.957755:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 275 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957767:0:64903:0:(rw26.c:912:ll_write_end()) pos 1126400, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957772:0:64903:0:(rw26.c:969:ll_write_end()) page@000000008feecd57[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957775:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000000b608db1 17ffffc0001001 3:0 ffff9b0342ae1798 275 lru 00000080:00200000:0.0:1769074822.957780:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000575f51b8 275: 1< 0 - - > 2< 1126400 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957782:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000008feecd57 00000080:00200000:0.0:1769074822.957783:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 20. 00000080:00200000:0.0:1769074822.957785:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 276 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957796:0:64903:0:(rw26.c:912:ll_write_end()) pos 1130496, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957800:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000793669f8[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957803:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000a0da4b7a 17ffffc0001001 3:0 ffff9b0342ae1af8 276 lru 00000080:00200000:0.0:1769074822.957809:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000001ecfa984 276: 1< 0 - - > 2< 1130496 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957811:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000793669f8 00000080:00200000:0.0:1769074822.957812:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 21. 00000080:00200000:0.0:1769074822.957814:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 277 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957825:0:64903:0:(rw26.c:912:ll_write_end()) pos 1134592, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957829:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000105183a8[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957832:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000076d29cd8 17ffffc0001001 3:0 ffff9b0342ae1d80 277 lru 00000080:00200000:0.0:1769074822.957837:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b3a1a975 277: 1< 0 - - > 2< 1134592 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957839:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000105183a8 00000080:00200000:0.0:1769074822.957840:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 22. 00000080:00200000:0.0:1769074822.957842:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 278 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957854:0:64903:0:(rw26.c:912:ll_write_end()) pos 1138688, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957858:0:64903:0:(rw26.c:969:ll_write_end()) page@000000008eec8ff6[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957861:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000074fed8e5 17ffffc0001001 3:0 ffff9b0342ae1e58 278 lru 00000080:00200000:0.0:1769074822.957865:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000477e7e69 278: 1< 0 - - > 2< 1138688 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957868:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000008eec8ff6 00000080:00200000:0.0:1769074822.957869:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 23. 00000080:00200000:0.0:1769074822.957871:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 279 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957882:0:64903:0:(rw26.c:912:ll_write_end()) pos 1142784, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957886:0:64903:0:(rw26.c:969:ll_write_end()) page@000000000b254a3f[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957889:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000000e004a17 17ffffc0001001 3:0 ffff9b0342ae1438 279 lru 00000080:00200000:0.0:1769074822.957895:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000002cd2ec61 279: 1< 0 - - > 2< 1142784 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957898:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000000b254a3f 00000080:00200000:0.0:1769074822.957899:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 24. 00000080:00200000:0.0:1769074822.957901:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 280 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957913:0:64903:0:(rw26.c:912:ll_write_end()) pos 1146880, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957917:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000141aacc9[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957920:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000b243e3e4 17ffffc0001001 3:0 ffff9b0342ae1510 280 lru 00000080:00200000:0.0:1769074822.957926:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b38621e1 280: 1< 0 - - > 2< 1146880 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957928:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000141aacc9 00000080:00200000:0.0:1769074822.957929:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 25. 00000080:00200000:0.0:1769074822.957931:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 281 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957943:0:64903:0:(rw26.c:912:ll_write_end()) pos 1150976, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957947:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000039ff9f2[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957950:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000001e95b980 17ffffc0001001 3:0 ffff9b0342ae1948 281 lru 00000080:00200000:0.0:1769074822.957955:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000196b5867 281: 1< 0 - - > 2< 1150976 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957957:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000039ff9f2 00000080:00200000:0.0:1769074822.957958:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 26. 00000080:00200000:0.0:1769074822.957960:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 282 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.957972:0:64903:0:(rw26.c:912:ll_write_end()) pos 1155072, len 4096, copied 4096 00000080:00200000:0.0:1769074822.957976:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000cae6f895[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.957979:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000004dcfb23 17ffffc0001001 3:0 ffff9b0342ae1000 282 lru 00000080:00200000:0.0:1769074822.957985:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000fb7d2a8f 282: 1< 0 - - > 2< 1155072 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.957987:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000cae6f895 00000080:00200000:0.0:1769074822.957988:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 27. 00000080:00200000:0.0:1769074822.957990:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 283 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958001:0:64903:0:(rw26.c:912:ll_write_end()) pos 1159168, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958005:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000099a0ce14[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958007:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000dbe34c78 17ffffc0001001 3:0 ffff9b0342ae1ca8 283 lru 00000080:00200000:0.0:1769074822.958013:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000ca24c831 283: 1< 0 - - > 2< 1159168 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958015:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000099a0ce14 00000080:00200000:0.0:1769074822.958016:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 28. 00000080:00200000:0.0:1769074822.958018:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 284 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958029:0:64903:0:(rw26.c:912:ll_write_end()) pos 1163264, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958033:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000b36c9f5f[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958035:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000776b3cf9 17ffffc0001001 3:0 ffff9b0342ae15e8 284 lru 00000080:00200000:0.0:1769074822.958041:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006b04fb1e 284: 1< 0 - - > 2< 1163264 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958043:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000b36c9f5f 00000080:00200000:0.0:1769074822.958044:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 29. 00000080:00200000:0.0:1769074822.958046:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 285 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958066:0:64903:0:(rw26.c:912:ll_write_end()) pos 1167360, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958071:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000720f87b4[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958073:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000fb2c7876 17ffffc0001021 2:0 ffff9b0342ae10d8 285 lru 00000080:00200000:0.0:1769074822.958079:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000008266595f 285: 1< 0 - - > 2< 1167360 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958081:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000720f87b4 00000080:00200000:0.0:1769074822.958082:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 30. 00000080:00200000:0.0:1769074822.958084:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 286 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958095:0:64903:0:(rw26.c:912:ll_write_end()) pos 1171456, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958099:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000cd70eec4[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958102:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000008ef0b9e7 17ffffc0001001 3:0 ffff9b0342ae1288 286 lru 00000080:00200000:0.0:1769074822.958107:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000fd6aae63 286: 1< 0 - - > 2< 1171456 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958110:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000cd70eec4 00000080:00200000:0.0:1769074822.958111:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 31. 00000080:00200000:0.0:1769074822.958113:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 287 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958126:0:64903:0:(rw26.c:912:ll_write_end()) pos 1175552, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958131:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000d9639aa5[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958134:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000005a3db382 17ffffc0001001 3:0 ffff9b0342ae1870 287 lru 00000080:00200000:0.0:1769074822.958139:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e6a491d5 287: 1< 0 - - > 2< 1175552 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958142:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000d9639aa5 00000080:00200000:0.0:1769074822.958143:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 32. 00000080:00200000:0.0:1769074822.958144:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 288 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958156:0:64903:0:(rw26.c:912:ll_write_end()) pos 1179648, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958160:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000f5c6d22c[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958163:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000aa015862 17ffffc0001001 3:0 ffff9b0342ae1bd0 288 lru 00000080:00200000:0.0:1769074822.958168:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e6350135 288: 1< 0 - - > 2< 1179648 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958170:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000f5c6d22c 00000080:00200000:0.0:1769074822.958171:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 33. 00000080:00200000:0.0:1769074822.958173:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 289 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958184:0:64903:0:(rw26.c:912:ll_write_end()) pos 1183744, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958188:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000050be4ed2[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958191:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000a7cc5748 17ffffc0001001 3:0 ffff9b0342ae16c0 289 lru 00000080:00200000:0.0:1769074822.958197:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000797ef8b3 289: 1< 0 - - > 2< 1183744 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958199:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000050be4ed2 00000080:00200000:0.0:1769074822.958200:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 34. 00000080:00200000:0.0:1769074822.958201:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 290 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958212:0:64903:0:(rw26.c:912:ll_write_end()) pos 1187840, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958216:0:64903:0:(rw26.c:969:ll_write_end()) page@000000008fe88765[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958219:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000a07f872b 17ffffc0001001 3:0 ffff9b0342ae1360 290 lru 00000080:00200000:0.0:1769074822.958224:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e1e1f474 290: 1< 0 - - > 2< 1187840 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958226:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000008fe88765 00000080:00200000:0.0:1769074822.958227:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 35. 00000080:00200000:0.0:1769074822.958229:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 291 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958292:0:64903:0:(rw26.c:912:ll_write_end()) pos 1191936, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958297:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000f5f5e8aa[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958300:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000918c78c7 17ffffc0001001 3:0 ffff9b0342ae1a20 291 lru 00000080:00200000:0.0:1769074822.958306:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b889afd7 291: 1< 0 - - > 2< 1191936 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958308:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000f5f5e8aa 00000080:00200000:0.0:1769074822.958309:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 36. 00000080:00200000:0.0:1769074822.958311:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 292 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958328:0:64903:0:(rw26.c:912:ll_write_end()) pos 1196032, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958332:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000688f82bc[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958335:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000000d151bc0 17ffffc0001001 3:0 ffff9b03515e9af8 292 lru 00000080:00200000:0.0:1769074822.958340:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000196678bd 292: 1< 0 - - > 2< 1196032 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958342:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000688f82bc 00000080:00200000:0.0:1769074822.958343:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 37. 00000080:00200000:0.0:1769074822.958344:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 293 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958358:0:64903:0:(rw26.c:912:ll_write_end()) pos 1200128, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958362:0:64903:0:(rw26.c:969:ll_write_end()) page@000000002233c099[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958365:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000008bbe9877 17ffffc0001001 3:0 ffff9b03515e9d80 293 lru 00000080:00200000:0.0:1769074822.958370:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000044652e40 293: 1< 0 - - > 2< 1200128 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958372:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000002233c099 00000080:00200000:0.0:1769074822.958373:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 38. 00000080:00200000:0.0:1769074822.958375:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 294 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958394:0:64903:0:(rw26.c:912:ll_write_end()) pos 1204224, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958398:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000079b30596[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958401:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000038ffd152 17ffffc0001001 3:0 ffff9b03515e9e58 294 lru 00000080:00200000:0.0:1769074822.958405:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000008c557e45 294: 1< 0 - - > 2< 1204224 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958407:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000079b30596 00000080:00200000:0.0:1769074822.958409:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 39. 00000080:00200000:0.0:1769074822.958410:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 295 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958422:0:64903:0:(rw26.c:912:ll_write_end()) pos 1208320, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958426:0:64903:0:(rw26.c:969:ll_write_end()) page@000000006673c3ff[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958428:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000070b9619b 17ffffc0001001 3:0 ffff9b03515e9438 295 lru 00000080:00200000:0.0:1769074822.958433:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000000d79f414 295: 1< 0 - - > 2< 1208320 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958435:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000006673c3ff 00000080:00200000:0.0:1769074822.958435:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 40. 00000080:00200000:0.0:1769074822.958437:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 296 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958451:0:64903:0:(rw26.c:912:ll_write_end()) pos 1212416, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958454:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000043567820[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958456:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ff0a37c2 17ffffc0001001 3:0 ffff9b03515e9510 296 lru 00000080:00200000:0.0:1769074822.958461:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000f9298729 296: 1< 0 - - > 2< 1212416 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958463:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000043567820 00000080:00200000:0.0:1769074822.958464:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 41. 00000080:00200000:0.0:1769074822.958466:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 297 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958479:0:64903:0:(rw26.c:912:ll_write_end()) pos 1216512, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958483:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000015729187[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958485:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000d25be520 17ffffc0001001 3:0 ffff9b03515e9948 297 lru 00000080:00200000:0.0:1769074822.958493:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000053c0aee4 297: 1< 0 - - > 2< 1216512 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958495:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000015729187 00000080:00200000:0.0:1769074822.958496:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 42. 00000080:00200000:0.0:1769074822.958498:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 298 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958510:0:64903:0:(rw26.c:912:ll_write_end()) pos 1220608, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958513:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000d712e4ca[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958515:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000006abbfd98 17ffffc0001001 3:0 ffff9b03515e9000 298 lru 00000080:00200000:0.0:1769074822.958521:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000acec10e7 298: 1< 0 - - > 2< 1220608 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958522:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000d712e4ca 00000080:00200000:0.0:1769074822.958523:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 43. 00000080:00200000:0.0:1769074822.958524:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 299 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958535:0:64903:0:(rw26.c:912:ll_write_end()) pos 1224704, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958539:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000068afeb37[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958542:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000368365ad 17ffffc0001001 3:0 ffff9b03515e9ca8 299 lru 00000080:00200000:0.0:1769074822.958546:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000d7881c39 299: 1< 0 - - > 2< 1224704 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958548:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000068afeb37 00000080:00200000:0.0:1769074822.958549:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 44. 00000080:00200000:0.0:1769074822.958551:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 300 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958566:0:64903:0:(rw26.c:912:ll_write_end()) pos 1228800, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958570:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000f9c58315[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958573:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000019344ee4 17ffffc0001021 2:0 ffff9b03515e95e8 300 lru 00000080:00200000:0.0:1769074822.958578:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006b7de426 300: 1< 0 - - > 2< 1228800 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958580:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000f9c58315 00000080:00200000:0.0:1769074822.958581:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 45. 00000080:00200000:0.0:1769074822.958583:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 301 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958598:0:64903:0:(rw26.c:912:ll_write_end()) pos 1232896, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958601:0:64903:0:(rw26.c:969:ll_write_end()) page@000000005260478b[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958603:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000000e673bb4 17ffffc0001001 3:0 ffff9b03515e90d8 301 lru 00000080:00200000:0.0:1769074822.958607:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000089cc5a1c 301: 1< 0 - - > 2< 1232896 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958609:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000005260478b 00000080:00200000:0.0:1769074822.958610:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 46. 00000080:00200000:0.0:1769074822.958611:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 302 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958625:0:64903:0:(rw26.c:912:ll_write_end()) pos 1236992, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958628:0:64903:0:(rw26.c:969:ll_write_end()) page@000000000b05402a[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958630:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000001595fb16 17ffffc0001001 3:0 ffff9b03515e9288 302 lru 00000080:00200000:0.0:1769074822.958635:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000441e4f71 302: 1< 0 - - > 2< 1236992 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958637:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000000b05402a 00000080:00200000:0.0:1769074822.958638:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 47. 00000080:00200000:0.0:1769074822.958640:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 303 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958651:0:64903:0:(rw26.c:912:ll_write_end()) pos 1241088, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958656:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000f4a903e4[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958658:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000008bb5852c 17ffffc0001001 3:0 ffff9b03515e9870 303 lru 00000080:00200000:0.0:1769074822.958662:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000003117b34b 303: 1< 0 - - > 2< 1241088 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958664:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000f4a903e4 00000080:00200000:0.0:1769074822.958664:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 48. 00000080:00200000:0.0:1769074822.958666:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 304 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958678:0:64903:0:(rw26.c:912:ll_write_end()) pos 1245184, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958681:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000034527f56[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958684:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000c21a4578 17ffffc0001001 3:0 ffff9b03515e9bd0 304 lru 00000080:00200000:0.0:1769074822.958688:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000284ba40b 304: 1< 0 - - > 2< 1245184 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958690:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000034527f56 00000080:00200000:0.0:1769074822.958690:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 49. 00000080:00200000:0.0:1769074822.958692:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 305 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958705:0:64903:0:(rw26.c:912:ll_write_end()) pos 1249280, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958708:0:64903:0:(rw26.c:969:ll_write_end()) page@000000009963abfe[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958710:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000e4b110dc 17ffffc0001001 3:0 ffff9b03515e96c0 305 lru 00000080:00200000:0.0:1769074822.958714:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000084b46573 305: 1< 0 - - > 2< 1249280 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958716:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000009963abfe 00000080:00200000:0.0:1769074822.958717:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 50. 00000080:00200000:0.0:1769074822.958718:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 306 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958730:0:64903:0:(rw26.c:912:ll_write_end()) pos 1253376, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958733:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000eb63e161[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958736:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000087356159 17ffffc0001001 3:0 ffff9b03515e9360 306 lru 00000080:00200000:0.0:1769074822.958740:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000005987ced3 306: 1< 0 - - > 2< 1253376 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958742:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000eb63e161 00000080:00200000:0.0:1769074822.958743:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 51. 00000080:00200000:0.0:1769074822.958745:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 307 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958757:0:64903:0:(rw26.c:912:ll_write_end()) pos 1257472, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958761:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000050b3d5a[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958764:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000059471112 17ffffc0001001 3:0 ffff9b03515e9a20 307 lru 00000080:00200000:0.0:1769074822.958768:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000355468cf 307: 1< 0 - - > 2< 1257472 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958771:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000050b3d5a 00000080:00200000:0.0:1769074822.958772:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 52. 00000080:00200000:0.0:1769074822.958774:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 308 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958790:0:64903:0:(rw26.c:912:ll_write_end()) pos 1261568, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958794:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000025dc9187[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958797:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000090916756 17ffffc0001001 3:0 ffff9b03515e91b0 308 lru 00000080:00200000:0.0:1769074822.958801:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000009d178ebd 308: 1< 0 - - > 2< 1261568 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958803:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000025dc9187 00000080:00200000:0.0:1769074822.958803:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 53. 00000080:00200000:0.0:1769074822.958805:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 309 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958817:0:64903:0:(rw26.c:912:ll_write_end()) pos 1265664, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958821:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000d6b34a59[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958824:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000e038fa18 17ffffc0001001 3:0 ffff9b03515e9798 309 lru 00000080:00200000:0.0:1769074822.958829:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000007c6f1407 309: 1< 0 - - > 2< 1265664 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958831:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000d6b34a59 00000080:00200000:0.0:1769074822.958832:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 54. 00000080:00200000:0.0:1769074822.958834:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 310 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958847:0:64903:0:(rw26.c:912:ll_write_end()) pos 1269760, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958851:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000085f06bc9[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958854:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000bdf547c0 17ffffc0001001 3:0 ffff9b036580daf8 310 lru 00000080:00200000:0.0:1769074822.958860:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000002d873773 310: 1< 0 - - > 2< 1269760 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958862:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000085f06bc9 00000080:00200000:0.0:1769074822.958863:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 55. 00000080:00200000:0.0:1769074822.958864:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 311 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958878:0:64903:0:(rw26.c:912:ll_write_end()) pos 1273856, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958883:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000027ac7270[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958885:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000aa995953 17ffffc0001001 3:0 ffff9b036580dd80 311 lru 00000080:00200000:0.0:1769074822.958889:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000007940e76b 311: 1< 0 - - > 2< 1273856 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958891:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000027ac7270 00000080:00200000:0.0:1769074822.958892:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 56. 00000080:00200000:0.0:1769074822.958894:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 312 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958906:0:64903:0:(rw26.c:912:ll_write_end()) pos 1277952, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958910:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000090eb87fc[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958912:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000003523f854 17ffffc0001001 3:0 ffff9b036580de58 312 lru 00000080:00200000:0.0:1769074822.958917:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006b7cb548 312: 1< 0 - - > 2< 1277952 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958919:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000090eb87fc 00000080:00200000:0.0:1769074822.958920:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 57. 00000080:00200000:0.0:1769074822.958922:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 313 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958934:0:64903:0:(rw26.c:912:ll_write_end()) pos 1282048, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958938:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000f3c6bb58[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958941:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000b96d6366 17ffffc0001001 3:0 ffff9b036580d438 313 lru 00000080:00200000:0.0:1769074822.958945:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000094152411 313: 1< 0 - - > 2< 1282048 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958947:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000f3c6bb58 00000080:00200000:0.0:1769074822.958948:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 58. 00000080:00200000:0.0:1769074822.958950:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 314 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958963:0:64903:0:(rw26.c:912:ll_write_end()) pos 1286144, len 4096, copied 4096 00000080:00200000:0.0:1769074822.958967:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000051d45235[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.958970:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000001b385042 17ffffc0001001 3:0 ffff9b036580d510 314 lru 00000080:00200000:0.0:1769074822.958975:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000013468b4b 314: 1< 0 - - > 2< 1286144 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.958977:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000051d45235 00000080:00200000:0.0:1769074822.958978:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 59. 00000080:00200000:0.0:1769074822.958980:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 315 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.958999:0:64903:0:(rw26.c:912:ll_write_end()) pos 1290240, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959003:0:64903:0:(rw26.c:969:ll_write_end()) page@000000007cbe4598[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959006:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000003d8b370f 17ffffc0001021 2:0 ffff9b036580d948 315 lru 00000080:00200000:0.0:1769074822.959011:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000007ac9dd84 315: 1< 0 - - > 2< 1290240 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959013:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000007cbe4598 00000080:00200000:0.0:1769074822.959014:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 60. 00000080:00200000:0.0:1769074822.959016:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 316 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959031:0:64903:0:(rw26.c:912:ll_write_end()) pos 1294336, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959035:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000bf3d2871[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959037:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000009b16251f 17ffffc0001001 3:0 ffff9b036580d000 316 lru 00000080:00200000:0.0:1769074822.959042:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000002cc9f61b 316: 1< 0 - - > 2< 1294336 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959044:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000bf3d2871 00000080:00200000:0.0:1769074822.959045:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 61. 00000080:00200000:0.0:1769074822.959046:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 317 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959060:0:64903:0:(rw26.c:912:ll_write_end()) pos 1298432, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959065:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000062bd0088[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959067:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000fbe29a78 17ffffc0001001 3:0 ffff9b036580dca8 317 lru 00000080:00200000:0.0:1769074822.959071:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000d7c44a09 317: 1< 0 - - > 2< 1298432 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959074:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000062bd0088 00000080:00200000:0.0:1769074822.959075:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 62. 00000080:00200000:0.0:1769074822.959077:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 318 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959091:0:64903:0:(rw26.c:912:ll_write_end()) pos 1302528, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959095:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000c71983b6[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959097:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000090ef5124 17ffffc0001001 3:0 ffff9b036580d5e8 318 lru 00000080:00200000:0.0:1769074822.959101:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000004557430b 318: 1< 0 - - > 2< 1302528 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959104:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000c71983b6 00000080:00200000:0.0:1769074822.959105:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 63. 00000080:00200000:0.0:1769074822.959106:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 319 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959119:0:64903:0:(rw26.c:912:ll_write_end()) pos 1306624, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959122:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000ec9235c6[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959125:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000008d0df64c 17ffffc0001001 3:0 ffff9b036580d0d8 319 lru 00000080:00200000:0.0:1769074822.959130:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000019113115 319: 1< 0 - - > 2< 1306624 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959131:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000ec9235c6 00000080:00200000:0.0:1769074822.959132:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 64. 00000080:00200000:0.0:1769074822.959134:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 320 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959154:0:64903:0:(rw26.c:912:ll_write_end()) pos 1310720, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959158:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000d0b4d3f5[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959160:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000b10416bd 17ffffc0001001 3:0 ffff9b036580d288 320 lru 00000080:00200000:0.0:1769074822.959165:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b2a1d326 320: 1< 0 - - > 2< 1310720 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959167:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000d0b4d3f5 00000080:00200000:0.0:1769074822.959169:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 65. 00000080:00200000:0.0:1769074822.959170:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 321 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959185:0:64903:0:(rw26.c:912:ll_write_end()) pos 1314816, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959189:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000f04f887c[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959191:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000679d702e 17ffffc0001001 3:0 ffff9b036580d870 321 lru 00000080:00200000:0.0:1769074822.959195:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e9ce6d01 321: 1< 0 - - > 2< 1314816 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959197:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000f04f887c 00000080:00200000:0.0:1769074822.959198:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 66. 00000080:00200000:0.0:1769074822.959200:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 322 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959213:0:64903:0:(rw26.c:912:ll_write_end()) pos 1318912, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959217:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000081250fd6[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959219:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000cb7c588b 17ffffc0001001 3:0 ffff9b036580dbd0 322 lru 00000080:00200000:0.0:1769074822.959224:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000064f21527 322: 1< 0 - - > 2< 1318912 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959226:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000081250fd6 00000080:00200000:0.0:1769074822.959226:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 67. 00000080:00200000:0.0:1769074822.959228:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 323 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959278:0:64903:0:(rw26.c:912:ll_write_end()) pos 1323008, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959283:0:64903:0:(rw26.c:969:ll_write_end()) page@000000002997e9bb[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959286:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000d79d57a0 17ffffc0001001 3:0 ffff9b036580d6c0 323 lru 00000080:00200000:0.0:1769074822.959291:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000097c81c94 323: 1< 0 - - > 2< 1323008 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959293:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000002997e9bb 00000080:00200000:0.0:1769074822.959294:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 68. 00000080:00200000:0.0:1769074822.959296:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 324 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959308:0:64903:0:(rw26.c:912:ll_write_end()) pos 1327104, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959312:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a7f09af5[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959315:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000022023210 17ffffc0001001 3:0 ffff9b036580d360 324 lru 00000080:00200000:0.0:1769074822.959319:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000079f991af 324: 1< 0 - - > 2< 1327104 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959321:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a7f09af5 00000080:00200000:0.0:1769074822.959322:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 69. 00000080:00200000:0.0:1769074822.959324:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 325 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959335:0:64903:0:(rw26.c:912:ll_write_end()) pos 1331200, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959339:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000c57981eb[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959342:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000bd6f6489 17ffffc0001001 3:0 ffff9b036580da20 325 lru 00000080:00200000:0.0:1769074822.959347:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006c9ae2fe 325: 1< 0 - - > 2< 1331200 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959349:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000c57981eb 00000080:00200000:0.0:1769074822.959350:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 70. 00000080:00200000:0.0:1769074822.959352:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 326 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959364:0:64903:0:(rw26.c:912:ll_write_end()) pos 1335296, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959368:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a13a70df[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959371:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000001422df3b 17ffffc0001001 3:0 ffff9b036580d1b0 326 lru 00000080:00200000:0.0:1769074822.959376:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000aa2f2afe 326: 1< 0 - - > 2< 1335296 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959379:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a13a70df 00000080:00200000:0.0:1769074822.959380:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 71. 00000080:00200000:0.0:1769074822.959381:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 327 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959394:0:64903:0:(rw26.c:912:ll_write_end()) pos 1339392, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959398:0:64903:0:(rw26.c:969:ll_write_end()) page@000000003effd076[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959401:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000f8d77d1b 17ffffc0001001 3:0 ffff9b036580d798 327 lru 00000080:00200000:0.0:1769074822.959406:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000040db5f47 327: 1< 0 - - > 2< 1339392 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959408:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000003effd076 00000080:00200000:0.0:1769074822.959409:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 72. 00000080:00200000:0.0:1769074822.959411:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 328 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959426:0:64903:0:(rw26.c:912:ll_write_end()) pos 1343488, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959431:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000cc00f969[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959434:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000032f36110 17ffffc0001001 3:0 ffff9b0352a46510 328 lru 00000080:00200000:0.0:1769074822.959439:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006c357154 328: 1< 0 - - > 2< 1343488 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959440:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000cc00f969 00000080:00200000:0.0:1769074822.959441:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 73. 00000080:00200000:0.0:1769074822.959443:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 329 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959458:0:64903:0:(rw26.c:912:ll_write_end()) pos 1347584, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959462:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000e588583d[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959465:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000054490c33 17ffffc0001001 3:0 ffff9b0352a46948 329 lru 00000080:00200000:0.0:1769074822.959469:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000a4210957 329: 1< 0 - - > 2< 1347584 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959472:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000e588583d 00000080:00200000:0.0:1769074822.959472:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 74. 00000080:00200000:0.0:1769074822.959474:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 330 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959491:0:64903:0:(rw26.c:912:ll_write_end()) pos 1351680, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959495:0:64903:0:(rw26.c:969:ll_write_end()) page@000000004de28b3f[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959498:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000cd31e096 17ffffc0001021 2:0 ffff9b0352a46000 330 lru 00000080:00200000:0.0:1769074822.959503:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000002d7d1f9a 330: 1< 0 - - > 2< 1351680 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959505:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000004de28b3f 00000080:00200000:0.0:1769074822.959506:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 75. 00000080:00200000:0.0:1769074822.959508:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 331 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959521:0:64903:0:(rw26.c:912:ll_write_end()) pos 1355776, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959525:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a387656a[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959528:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000eaa03c37 17ffffc0001001 3:0 ffff9b0352a46ca8 331 lru 00000080:00200000:0.0:1769074822.959532:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000001aeb9e3c 331: 1< 0 - - > 2< 1355776 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959534:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a387656a 00000080:00200000:0.0:1769074822.959535:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 76. 00000080:00200000:0.0:1769074822.959537:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 332 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959548:0:64903:0:(rw26.c:912:ll_write_end()) pos 1359872, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959551:0:64903:0:(rw26.c:969:ll_write_end()) page@000000000fe7210d[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959553:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000fc106506 17ffffc0001001 3:0 ffff9b0352a465e8 332 lru 00000080:00200000:0.0:1769074822.959557:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000024e23b80 332: 1< 0 - - > 2< 1359872 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959559:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000000fe7210d 00000080:00200000:0.0:1769074822.959560:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 77. 00000080:00200000:0.0:1769074822.959561:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 333 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959574:0:64903:0:(rw26.c:912:ll_write_end()) pos 1363968, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959578:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000058075929[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959580:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000cf3f9d08 17ffffc0001001 3:0 ffff9b0352a460d8 333 lru 00000080:00200000:0.0:1769074822.959584:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000009f1701d8 333: 1< 0 - - > 2< 1363968 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959586:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000058075929 00000080:00200000:0.0:1769074822.959587:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 78. 00000080:00200000:0.0:1769074822.959588:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 334 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959600:0:64903:0:(rw26.c:912:ll_write_end()) pos 1368064, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959603:0:64903:0:(rw26.c:969:ll_write_end()) page@000000009a795f78[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959606:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000005c59fce6 17ffffc0001001 3:0 ffff9b0352a46288 334 lru 00000080:00200000:0.0:1769074822.959610:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000009c2d5a2 334: 1< 0 - - > 2< 1368064 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959613:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000009a795f78 00000080:00200000:0.0:1769074822.959614:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 79. 00000080:00200000:0.0:1769074822.959616:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 335 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959638:0:64903:0:(rw26.c:912:ll_write_end()) pos 1372160, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959642:0:64903:0:(rw26.c:969:ll_write_end()) page@000000002debdf3e[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959645:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000006b55b23a 17ffffc0001001 3:0 ffff9b0352a46870 335 lru 00000080:00200000:0.0:1769074822.959649:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000001e79c090 335: 1< 0 - - > 2< 1372160 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959652:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000002debdf3e 00000080:00200000:0.0:1769074822.959653:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 80. 00000080:00200000:0.0:1769074822.959654:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 336 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959669:0:64903:0:(rw26.c:912:ll_write_end()) pos 1376256, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959672:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a696bf8b[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959675:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000000fd40110 17ffffc0001001 3:0 ffff9b0352a46bd0 336 lru 00000080:00200000:0.0:1769074822.959681:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000097ddc02f 336: 1< 0 - - > 2< 1376256 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959683:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a696bf8b 00000080:00200000:0.0:1769074822.959684:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 81. 00000080:00200000:0.0:1769074822.959686:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 337 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959698:0:64903:0:(rw26.c:912:ll_write_end()) pos 1380352, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959701:0:64903:0:(rw26.c:969:ll_write_end()) page@000000003ed01794[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959704:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000055f63116 17ffffc0001001 3:0 ffff9b0352a466c0 337 lru 00000080:00200000:0.0:1769074822.959708:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000009eb504d3 337: 1< 0 - - > 2< 1380352 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959710:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000003ed01794 00000080:00200000:0.0:1769074822.959711:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 82. 00000080:00200000:0.0:1769074822.959713:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 338 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959726:0:64903:0:(rw26.c:912:ll_write_end()) pos 1384448, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959729:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000001026426[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959732:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000002bed9bab 17ffffc0001001 3:0 ffff9b0352a46360 338 lru 00000080:00200000:0.0:1769074822.959737:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000065655a6a 338: 1< 0 - - > 2< 1384448 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959739:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000001026426 00000080:00200000:0.0:1769074822.959741:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 83. 00000080:00200000:0.0:1769074822.959742:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 339 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959756:0:64903:0:(rw26.c:912:ll_write_end()) pos 1388544, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959760:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000085b8d8e1[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959768:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000086dce0b4 17ffffc0001001 3:0 ffff9b0352a46a20 339 lru 00000080:00200000:0.0:1769074822.959773:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000070cb65de 339: 1< 0 - - > 2< 1388544 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959775:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000085b8d8e1 00000080:00200000:0.0:1769074822.959776:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 84. 00000080:00200000:0.0:1769074822.959779:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 340 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959795:0:64903:0:(rw26.c:912:ll_write_end()) pos 1392640, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959798:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000bda0b383[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959800:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000d4e148c8 17ffffc0001001 3:0 ffff9b0352a461b0 340 lru 00000080:00200000:0.0:1769074822.959805:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000069b9cdce 340: 1< 0 - - > 2< 1392640 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959807:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000bda0b383 00000080:00200000:0.0:1769074822.959808:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 85. 00000080:00200000:0.0:1769074822.959810:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 341 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959823:0:64903:0:(rw26.c:912:ll_write_end()) pos 1396736, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959826:0:64903:0:(rw26.c:969:ll_write_end()) page@000000007c259a8b[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959828:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000006aa57b1f 17ffffc0001001 3:0 ffff9b0352a46798 341 lru 00000080:00200000:0.0:1769074822.959833:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000026e125ad 341: 1< 0 - - > 2< 1396736 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959836:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000007c259a8b 00000080:00200000:0.0:1769074822.959837:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 86. 00000080:00200000:0.0:1769074822.959838:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 342 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959852:0:64903:0:(rw26.c:912:ll_write_end()) pos 1400832, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959856:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000c5eb4545[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959858:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000005865d748 17ffffc0001001 3:0 ffff9b0352a46af8 342 lru 00000080:00200000:0.0:1769074822.959862:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000f1938dd2 342: 1< 0 - - > 2< 1400832 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959864:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000c5eb4545 00000080:00200000:0.0:1769074822.959865:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 87. 00000080:00200000:0.0:1769074822.959867:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 343 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959881:0:64903:0:(rw26.c:912:ll_write_end()) pos 1404928, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959885:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000042486f80[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959887:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000bbe2c2e9 17ffffc0001001 3:0 ffff9b0352a46d80 343 lru 00000080:00200000:0.0:1769074822.959891:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000630bd16d 343: 1< 0 - - > 2< 1404928 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959893:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000042486f80 00000080:00200000:0.0:1769074822.959894:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 88. 00000080:00200000:0.0:1769074822.959895:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 344 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959907:0:64903:0:(rw26.c:912:ll_write_end()) pos 1409024, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959910:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000c9b4d534[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959912:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000689c218f 17ffffc0001001 3:0 ffff9b0352a46e58 344 lru 00000080:00200000:0.0:1769074822.959916:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000a64d9848 344: 1< 0 - - > 2< 1409024 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959918:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000c9b4d534 00000080:00200000:0.0:1769074822.959919:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 89. 00000080:00200000:0.0:1769074822.959921:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 345 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959937:0:64903:0:(rw26.c:912:ll_write_end()) pos 1413120, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959942:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000af5c8736[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959944:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000008aea4bb0 17ffffc0001021 2:0 ffff9b0352a46438 345 lru 00000080:00200000:0.0:1769074822.959949:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000abb052a5 345: 1< 0 - - > 2< 1413120 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959950:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000af5c8736 00000080:00200000:0.0:1769074822.959951:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 90. 00000080:00200000:0.0:1769074822.959953:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 346 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959966:0:64903:0:(rw26.c:912:ll_write_end()) pos 1417216, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959970:0:64903:0:(rw26.c:969:ll_write_end()) page@000000008f943b1c[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.959973:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000005330b18c 17ffffc0001001 3:0 ffff9b035086d000 346 lru 00000080:00200000:0.0:1769074822.959977:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000db543d50 346: 1< 0 - - > 2< 1417216 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.959979:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000008f943b1c 00000080:00200000:0.0:1769074822.959980:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 91. 00000080:00200000:0.0:1769074822.959982:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 347 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.959995:0:64903:0:(rw26.c:912:ll_write_end()) pos 1421312, len 4096, copied 4096 00000080:00200000:0.0:1769074822.959999:0:64903:0:(rw26.c:969:ll_write_end()) page@000000004e0612f0[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.960002:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000fc806b95 17ffffc0001001 3:0 ffff9b035086dca8 347 lru 00000080:00200000:0.0:1769074822.960007:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000001b4a1819 347: 1< 0 - - > 2< 1421312 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.960009:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000004e0612f0 00000080:00200000:0.0:1769074822.960010:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 92. 00000080:00200000:0.0:1769074822.960012:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 348 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.960023:0:64903:0:(rw26.c:912:ll_write_end()) pos 1425408, len 4096, copied 4096 00000080:00200000:0.0:1769074822.960026:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000080b661b5[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.960029:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000364e44c7 17ffffc0001001 3:0 ffff9b035086d5e8 348 lru 00000080:00200000:0.0:1769074822.960033:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000fd92c1b7 348: 1< 0 - - > 2< 1425408 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.960035:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000080b661b5 00000080:00200000:0.0:1769074822.960036:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 93. 00000080:00200000:0.0:1769074822.960038:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 349 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.960050:0:64903:0:(rw26.c:912:ll_write_end()) pos 1429504, len 4096, copied 4096 00000080:00200000:0.0:1769074822.960054:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000be55cb05[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.960057:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000050fb6aaf 17ffffc0001001 3:0 ffff9b035086d0d8 349 lru 00000080:00200000:0.0:1769074822.960062:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000715fbef4 349: 1< 0 - - > 2< 1429504 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.960064:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000be55cb05 00000080:00200000:0.0:1769074822.960065:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 94. 00000080:00200000:0.0:1769074822.960066:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 350 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.960080:0:64903:0:(rw26.c:912:ll_write_end()) pos 1433600, len 4096, copied 4096 00000080:00200000:0.0:1769074822.960083:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000069b43188[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.960086:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000cbd5a904 17ffffc0001001 3:0 ffff9b035086d288 350 lru 00000080:00200000:0.0:1769074822.960090:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000a21d319e 350: 1< 0 - - > 2< 1433600 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.960093:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000069b43188 00000080:00200000:0.0:1769074822.960094:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 95. 00000080:00200000:0.0:1769074822.960095:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 351 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.960108:0:64903:0:(rw26.c:912:ll_write_end()) pos 1437696, len 4096, copied 4096 00000080:00200000:0.0:1769074822.960112:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000ebd6d55d[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.960114:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000a0dfa1dc 17ffffc0001001 3:0 ffff9b035086d870 351 lru 00000080:00200000:0.0:1769074822.960120:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006cf182dc 351: 1< 0 - - > 2< 1437696 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.960122:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000ebd6d55d 00000080:00200000:0.0:1769074822.960123:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 96. 00000080:00200000:0.0:1769074822.960124:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 352 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.960137:0:64903:0:(rw26.c:912:ll_write_end()) pos 1441792, len 4096, copied 4096 00000080:00200000:0.0:1769074822.960142:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000e5f83a88[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.960145:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000281cb401 17ffffc0001001 3:0 ffff9b035086dbd0 352 lru 00000080:00200000:0.0:1769074822.960149:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b896858d 352: 1< 0 - - > 2< 1441792 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.960151:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000e5f83a88 00000080:00200000:0.0:1769074822.960152:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 97. 00000080:00200000:0.0:1769074822.960154:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 353 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.960167:0:64903:0:(rw26.c:912:ll_write_end()) pos 1445888, len 4096, copied 4096 00000080:00200000:0.0:1769074822.960171:0:64903:0:(rw26.c:969:ll_write_end()) page@000000007e172612[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.960174:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000004289e60e 17ffffc0001001 3:0 ffff9b035086d6c0 353 lru 00000080:00200000:0.0:1769074822.960179:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000c195879f 353: 1< 0 - - > 2< 1445888 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.960181:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000007e172612 00000080:00200000:0.0:1769074822.960182:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 98. 00000080:00200000:0.0:1769074822.960184:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 354 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.960197:0:64903:0:(rw26.c:912:ll_write_end()) pos 1449984, len 4096, copied 4096 00000080:00200000:0.0:1769074822.960201:0:64903:0:(rw26.c:969:ll_write_end()) page@000000002cda51cd[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.960203:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000949b0ab8 17ffffc0001001 3:0 ffff9b035086d360 354 lru 00000080:00200000:0.0:1769074822.960208:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000085b8f334 354: 1< 0 - - > 2< 1449984 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.960211:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000002cda51cd 00000080:00200000:0.0:1769074822.960212:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 99. 00000080:00200000:0.0:1769074822.960214:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 355 of 0 to 4096 bytes 00000080:00200000:0.0:1769074822.960231:0:64903:0:(rw26.c:912:ll_write_end()) pos 1454080, len 4096, copied 4096 00000080:00200000:0.0:1769074822.960234:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000b4a1534d[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:0.0:1769074822.960237:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000870ce421 17ffffc0001001 3:0 ffff9b035086da20 355 lru 00000080:00200000:0.0:1769074822.964683:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000599f9dc3 355: 1< 0 - - > 2< 1454080 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:0.0:1769074822.964696:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000b4a1534d 00000080:00200000:0.0:1769074822.964699:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 100. 00000080:00200000:1.0:1769074822.971608:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000000db3100f 17ffffc0001001 3:0 ffff9b03439d2ca8 404 lru 00000080:00200000:1.0:1769074822.971614:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b28a1af0 404: 1< 0 - - > 2< 1654784 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971616:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000de0bdcc1 00000080:00200000:1.0:1769074822.971617:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 149. 00000080:00200000:1.0:1769074822.971619:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 405 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971635:0:64903:0:(rw26.c:912:ll_write_end()) pos 1658880, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971639:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000061382569[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971642:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000007e63d586 17ffffc0001001 3:0 ffff9b03439d25e8 405 lru 00000080:00200000:1.0:1769074822.971647:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000d7bcc4c4 405: 1< 0 - - > 2< 1658880 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971649:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000061382569 00000080:00200000:1.0:1769074822.971650:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 150. 00000080:00200000:1.0:1769074822.971652:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 406 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971667:0:64903:0:(rw26.c:912:ll_write_end()) pos 1662976, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971672:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000085e20c7d[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971674:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000011bab2a3 17ffffc0001001 3:0 ffff9b03439d20d8 406 lru 00000080:00200000:1.0:1769074822.971680:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000766a52ea 406: 1< 0 - - > 2< 1662976 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971683:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000085e20c7d 00000080:00200000:1.0:1769074822.971684:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 151. 00000080:00200000:1.0:1769074822.971686:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 407 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971699:0:64903:0:(rw26.c:912:ll_write_end()) pos 1667072, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971703:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000386f89e4[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971707:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000007198a0ca 17ffffc0001001 3:0 ffff9b03439d2288 407 lru 00000080:00200000:1.0:1769074822.971713:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000aff3debf 407: 1< 0 - - > 2< 1667072 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971715:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000386f89e4 00000080:00200000:1.0:1769074822.971716:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 152. 00000080:00200000:1.0:1769074822.971718:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 408 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971732:0:64903:0:(rw26.c:912:ll_write_end()) pos 1671168, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971736:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a13cb072[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971739:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000049c67c11 17ffffc0001001 3:0 ffff9b03439d2870 408 lru 00000080:00200000:1.0:1769074822.971744:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000001d09c0a3 408: 1< 0 - - > 2< 1671168 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971746:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a13cb072 00000080:00200000:1.0:1769074822.971747:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 153. 00000080:00200000:1.0:1769074822.971750:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 409 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971767:0:64903:0:(rw26.c:912:ll_write_end()) pos 1675264, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971772:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a2c94bbd[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971774:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000072807e65 17ffffc0001001 3:0 ffff9b03439d2bd0 409 lru 00000080:00200000:1.0:1769074822.971780:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000008a2f9984 409: 1< 0 - - > 2< 1675264 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971782:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a2c94bbd 00000080:00200000:1.0:1769074822.971783:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 154. 00000080:00200000:1.0:1769074822.971785:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 410 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971801:0:64903:0:(rw26.c:912:ll_write_end()) pos 1679360, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971806:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000015595314[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971809:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ada402d9 17ffffc0001001 3:0 ffff9b036598fca8 410 lru 00000080:00200000:1.0:1769074822.971815:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000dafbb416 410: 1< 0 - - > 2< 1679360 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971818:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000015595314 00000080:00200000:1.0:1769074822.971819:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 155. 00000080:00200000:1.0:1769074822.971821:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 411 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971835:0:64903:0:(rw26.c:912:ll_write_end()) pos 1683456, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971839:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000072b92a75[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971842:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000001f267905 17ffffc0001001 3:0 ffff9b036598f5e8 411 lru 00000080:00200000:1.0:1769074822.971848:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000025aec444 411: 1< 0 - - > 2< 1683456 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971850:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000072b92a75 00000080:00200000:1.0:1769074822.971851:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 156. 00000080:00200000:1.0:1769074822.971853:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 412 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971867:0:64903:0:(rw26.c:912:ll_write_end()) pos 1687552, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971872:0:64903:0:(rw26.c:969:ll_write_end()) page@000000005670a730[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971875:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000025a6c448 17ffffc0001001 3:0 ffff9b036598f0d8 412 lru 00000080:00200000:1.0:1769074822.971880:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000ba64f753 412: 1< 0 - - > 2< 1687552 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971882:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000005670a730 00000080:00200000:1.0:1769074822.971883:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 157. 00000080:00200000:1.0:1769074822.971885:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 413 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971900:0:64903:0:(rw26.c:912:ll_write_end()) pos 1691648, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971904:0:64903:0:(rw26.c:969:ll_write_end()) page@000000009ac44be6[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971907:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000186ab906 17ffffc0001001 3:0 ffff9b036598f288 413 lru 00000080:00200000:1.0:1769074822.971913:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006096625c 413: 1< 0 - - > 2< 1691648 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971915:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000009ac44be6 00000080:00200000:1.0:1769074822.971916:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 158. 00000080:00200000:1.0:1769074822.971918:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 414 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971938:0:64903:0:(rw26.c:912:ll_write_end()) pos 1695744, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971942:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000041f18840[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971945:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000397804e2 17ffffc0001001 3:0 ffff9b036598f870 414 lru 00000080:00200000:1.0:1769074822.971950:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000132f69bd 414: 1< 0 - - > 2< 1695744 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971952:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000041f18840 00000080:00200000:1.0:1769074822.971953:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 159. 00000080:00200000:1.0:1769074822.971955:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 415 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.971975:0:64903:0:(rw26.c:912:ll_write_end()) pos 1699840, len 4096, copied 4096 00000080:00200000:1.0:1769074822.971980:0:64903:0:(rw26.c:969:ll_write_end()) page@000000001b294fe5[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.971983:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000d9392e0d 17ffffc0001021 2:0 ffff9b036598fbd0 415 lru 00000080:00200000:1.0:1769074822.971988:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000002064ff81 415: 1< 0 - - > 2< 1699840 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.971990:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000001b294fe5 00000080:00200000:1.0:1769074822.971991:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 160. 00000080:00200000:1.0:1769074822.971993:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 416 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.972010:0:64903:0:(rw26.c:912:ll_write_end()) pos 1703936, len 4096, copied 4096 00000080:00200000:1.0:1769074822.972015:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000ada5984b[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.972018:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000048b90619 17ffffc0001001 3:0 ffff9b036598f6c0 416 lru 00000080:00200000:1.0:1769074822.972024:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000a27a1ed4 416: 1< 0 - - > 2< 1703936 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.972026:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000ada5984b 00000080:00200000:1.0:1769074822.972028:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 161. 00000080:00200000:1.0:1769074822.972030:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 417 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.972043:0:64903:0:(rw26.c:912:ll_write_end()) pos 1708032, len 4096, copied 4096 00000080:00200000:1.0:1769074822.972047:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000b7d7935e[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.972051:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000a3a7a3a9 17ffffc0001001 3:0 ffff9b036598f360 417 lru 00000080:00200000:1.0:1769074822.972057:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006295eefc 417: 1< 0 - - > 2< 1708032 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.972059:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000b7d7935e 00000080:00200000:1.0:1769074822.972060:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 162. 00000080:00200000:1.0:1769074822.972062:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 418 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.972808:0:64903:0:(rw26.c:912:ll_write_end()) pos 1712128, len 4096, copied 4096 00000080:00200000:1.0:1769074822.972816:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a9bdb6b9[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.972819:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000002ebd044b 17ffffc0001001 3:0 ffff9b036598fa20 418 lru 00000080:00200000:1.0:1769074822.972824:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000abd6b6c4 418: 1< 0 - - > 2< 1712128 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.972826:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a9bdb6b9 00000080:00200000:1.0:1769074822.972827:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 163. 00000080:00200000:1.0:1769074822.972829:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 419 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.972848:0:64903:0:(rw26.c:912:ll_write_end()) pos 1716224, len 4096, copied 4096 00000080:00200000:1.0:1769074822.972852:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000017919fcc[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.972855:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000d773b89f 17ffffc0001001 3:0 ffff9b036598f1b0 419 lru 00000080:00200000:1.0:1769074822.972860:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000c82ac447 419: 1< 0 - - > 2< 1716224 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.972862:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000017919fcc 00000080:00200000:1.0:1769074822.972863:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 164. 00000080:00200000:1.0:1769074822.972865:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 420 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.972877:0:64903:0:(rw26.c:912:ll_write_end()) pos 1720320, len 4096, copied 4096 00000080:00200000:1.0:1769074822.972881:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000c6be1fde[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.972884:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000047748d0 17ffffc0001001 3:0 ffff9b036598f798 420 lru 00000080:00200000:1.0:1769074822.972889:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000662de425 420: 1< 0 - - > 2< 1720320 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.972891:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000c6be1fde 00000080:00200000:1.0:1769074822.972891:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 165. 00000080:00200000:1.0:1769074822.972893:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 421 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.972907:0:64903:0:(rw26.c:912:ll_write_end()) pos 1724416, len 4096, copied 4096 00000080:00200000:1.0:1769074822.972911:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000080e8cd25[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.972914:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000072845bfd 17ffffc0001001 3:0 ffff9b036598faf8 421 lru 00000080:00200000:1.0:1769074822.972918:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000c9892941 421: 1< 0 - - > 2< 1724416 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.972921:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000080e8cd25 00000080:00200000:1.0:1769074822.972922:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 166. 00000080:00200000:1.0:1769074822.972923:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 422 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.972937:0:64903:0:(rw26.c:912:ll_write_end()) pos 1728512, len 4096, copied 4096 00000080:00200000:1.0:1769074822.972941:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a2302064[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.972943:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000004afc9cd8 17ffffc0001001 3:0 ffff9b036598fd80 422 lru 00000080:00200000:1.0:1769074822.972948:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000222b264c 422: 1< 0 - - > 2< 1728512 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.972949:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a2302064 00000080:00200000:1.0:1769074822.972950:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 167. 00000080:00200000:1.0:1769074822.972952:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 423 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.972967:0:64903:0:(rw26.c:912:ll_write_end()) pos 1732608, len 4096, copied 4096 00000080:00200000:1.0:1769074822.972971:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000be23fd61[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.972974:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000099aabc7c 17ffffc0001001 3:0 ffff9b036598fe58 423 lru 00000080:00200000:1.0:1769074822.972979:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000003cd66801 423: 1< 0 - - > 2< 1732608 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.972980:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000be23fd61 00000080:00200000:1.0:1769074822.972981:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 168. 00000080:00200000:1.0:1769074822.972982:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 424 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.972995:0:64903:0:(rw26.c:912:ll_write_end()) pos 1736704, len 4096, copied 4096 00000080:00200000:1.0:1769074822.972998:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000083d3c83b[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.973001:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000097eee0e8 17ffffc0001001 3:0 ffff9b036598f438 424 lru 00000080:00200000:1.0:1769074822.973006:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000144e50ec 424: 1< 0 - - > 2< 1736704 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.973009:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000083d3c83b 00000080:00200000:1.0:1769074822.973010:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 169. 00000080:00200000:1.0:1769074822.973011:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 425 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.973023:0:64903:0:(rw26.c:912:ll_write_end()) pos 1740800, len 4096, copied 4096 00000080:00200000:1.0:1769074822.973027:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000c6710691[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.973029:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000f7d22a8e 17ffffc0001001 3:0 ffff9b036598f510 425 lru 00000080:00200000:1.0:1769074822.973033:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000001c4aa86e 425: 1< 0 - - > 2< 1740800 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.973035:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000c6710691 00000080:00200000:1.0:1769074822.973036:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 170. 00000080:00200000:1.0:1769074822.973037:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 426 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.973049:0:64903:0:(rw26.c:912:ll_write_end()) pos 1744896, len 4096, copied 4096 00000080:00200000:1.0:1769074822.973053:0:64903:0:(rw26.c:969:ll_write_end()) page@000000005fce7136[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.973057:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000f0823ea9 17ffffc0001001 3:0 ffff9b036598f948 426 lru 00000080:00200000:1.0:1769074822.973062:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000009cb320ce 426: 1< 0 - - > 2< 1744896 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.973064:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000005fce7136 00000080:00200000:1.0:1769074822.973065:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 171. 00000080:00200000:1.0:1769074822.973067:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 427 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974546:0:64903:0:(rw26.c:912:ll_write_end()) pos 1748992, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974560:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000032259c80[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974563:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000015edbed0 17ffffc0001001 3:0 ffff9b036598f000 427 lru 00000080:00200000:1.0:1769074822.974569:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000f2a6c594 427: 1< 0 - - > 2< 1748992 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974572:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000032259c80 00000080:00200000:1.0:1769074822.974573:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 172. 00000080:00200000:1.0:1769074822.974576:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 428 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974595:0:64903:0:(rw26.c:912:ll_write_end()) pos 1753088, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974599:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000e28ffc1c[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974602:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000007050d712 17ffffc0001001 3:0 ffff9b036598c948 428 lru 00000080:00200000:1.0:1769074822.974607:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000000cf54204 428: 1< 0 - - > 2< 1753088 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974609:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000e28ffc1c 00000080:00200000:1.0:1769074822.974610:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 173. 00000080:00200000:1.0:1769074822.974612:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 429 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974631:0:64903:0:(rw26.c:912:ll_write_end()) pos 1757184, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974634:0:64903:0:(rw26.c:969:ll_write_end()) page@000000007f771077[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974637:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000a88ce2dc 17ffffc0001001 3:0 ffff9b036598c000 429 lru 00000080:00200000:1.0:1769074822.974641:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b4455ecf 429: 1< 0 - - > 2< 1757184 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974643:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000007f771077 00000080:00200000:1.0:1769074822.974644:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 174. 00000080:00200000:1.0:1769074822.974646:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 430 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974666:0:64903:0:(rw26.c:912:ll_write_end()) pos 1761280, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974670:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000880680ab[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974673:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000308590cd 17ffffc0001021 2:0 ffff9b036598cca8 430 lru 00000080:00200000:1.0:1769074822.974678:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000005b24ce73 430: 1< 0 - - > 2< 1761280 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974680:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000880680ab 00000080:00200000:1.0:1769074822.974681:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 175. 00000080:00200000:1.0:1769074822.974683:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 431 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974698:0:64903:0:(rw26.c:912:ll_write_end()) pos 1765376, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974701:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000b061c373[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974704:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000b2deefd0 17ffffc0001001 3:0 ffff9b036598c5e8 431 lru 00000080:00200000:1.0:1769074822.974709:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000a4562007 431: 1< 0 - - > 2< 1765376 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974711:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000b061c373 00000080:00200000:1.0:1769074822.974712:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 176. 00000080:00200000:1.0:1769074822.974714:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 432 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974726:0:64903:0:(rw26.c:912:ll_write_end()) pos 1769472, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974729:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000ea9e90bb[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974732:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000035d047bc 17ffffc0001001 3:0 ffff9b036598c0d8 432 lru 00000080:00200000:1.0:1769074822.974737:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000074822d4f 432: 1< 0 - - > 2< 1769472 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974739:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000ea9e90bb 00000080:00200000:1.0:1769074822.974740:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 177. 00000080:00200000:1.0:1769074822.974742:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 433 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974755:0:64903:0:(rw26.c:912:ll_write_end()) pos 1773568, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974758:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000effa3cb7[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974761:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000008784c908 17ffffc0001001 3:0 ffff9b036598c288 433 lru 00000080:00200000:1.0:1769074822.974765:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000009c8cefbb 433: 1< 0 - - > 2< 1773568 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974767:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000effa3cb7 00000080:00200000:1.0:1769074822.974768:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 178. 00000080:00200000:1.0:1769074822.974770:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 434 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974785:0:64903:0:(rw26.c:912:ll_write_end()) pos 1777664, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974788:0:64903:0:(rw26.c:969:ll_write_end()) page@000000007a677855[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974791:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ed0851d1 17ffffc0001001 3:0 ffff9b036598c870 434 lru 00000080:00200000:1.0:1769074822.974795:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000003df558cc 434: 1< 0 - - > 2< 1777664 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974797:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000007a677855 00000080:00200000:1.0:1769074822.974798:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 179. 00000080:00200000:1.0:1769074822.974800:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 435 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974812:0:64903:0:(rw26.c:912:ll_write_end()) pos 1781760, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974815:0:64903:0:(rw26.c:969:ll_write_end()) page@000000007b45cb27[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974818:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000009fed26d4 17ffffc0001001 3:0 ffff9b036598cbd0 435 lru 00000080:00200000:1.0:1769074822.974822:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e22a5100 435: 1< 0 - - > 2< 1781760 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974824:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000007b45cb27 00000080:00200000:1.0:1769074822.974825:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 180. 00000080:00200000:1.0:1769074822.974827:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 436 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974841:0:64903:0:(rw26.c:912:ll_write_end()) pos 1785856, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974844:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000c7fe2450[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974847:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000b28e92fb 17ffffc0001001 3:0 ffff9b036598c6c0 436 lru 00000080:00200000:1.0:1769074822.974852:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000000368c970 436: 1< 0 - - > 2< 1785856 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974854:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000c7fe2450 00000080:00200000:1.0:1769074822.974855:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 181. 00000080:00200000:1.0:1769074822.974858:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 437 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974880:0:64903:0:(rw26.c:912:ll_write_end()) pos 1789952, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974884:0:64903:0:(rw26.c:969:ll_write_end()) page@000000004a710b6b[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974887:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000007c91c1a1 17ffffc0001001 3:0 ffff9b036598c360 437 lru 00000080:00200000:1.0:1769074822.974891:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000005d72b2f 437: 1< 0 - - > 2< 1789952 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974893:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000004a710b6b 00000080:00200000:1.0:1769074822.974894:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 182. 00000080:00200000:1.0:1769074822.974896:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 438 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974908:0:64903:0:(rw26.c:912:ll_write_end()) pos 1794048, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974912:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000bd7fb40c[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974914:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000005eac2b32 17ffffc0001001 3:0 ffff9b036598ca20 438 lru 00000080:00200000:1.0:1769074822.974919:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000078e2af3 438: 1< 0 - - > 2< 1794048 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974921:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000bd7fb40c 00000080:00200000:1.0:1769074822.974922:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 183. 00000080:00200000:1.0:1769074822.974924:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 439 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974938:0:64903:0:(rw26.c:912:ll_write_end()) pos 1798144, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974942:0:64903:0:(rw26.c:969:ll_write_end()) page@000000003e8fc322[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974944:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000067886abc 17ffffc0001001 3:0 ffff9b036598c1b0 439 lru 00000080:00200000:1.0:1769074822.974949:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000f9911f41 439: 1< 0 - - > 2< 1798144 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974951:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000003e8fc322 00000080:00200000:1.0:1769074822.974952:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 184. 00000080:00200000:1.0:1769074822.974953:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 440 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974966:0:64903:0:(rw26.c:912:ll_write_end()) pos 1802240, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974970:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000de6a4cc9[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.974973:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000e4f5355b 17ffffc0001001 3:0 ffff9b036598c798 440 lru 00000080:00200000:1.0:1769074822.974978:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000000b5374de 440: 1< 0 - - > 2< 1802240 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.974980:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000de6a4cc9 00000080:00200000:1.0:1769074822.974981:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 185. 00000080:00200000:1.0:1769074822.974983:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 441 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.974995:0:64903:0:(rw26.c:912:ll_write_end()) pos 1806336, len 4096, copied 4096 00000080:00200000:1.0:1769074822.974999:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000b7c6ddc3[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975001:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000066229028 17ffffc0001001 3:0 ffff9b036598caf8 441 lru 00000080:00200000:1.0:1769074822.975005:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b2c60ec0 441: 1< 0 - - > 2< 1806336 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975007:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000b7c6ddc3 00000080:00200000:1.0:1769074822.975008:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 186. 00000080:00200000:1.0:1769074822.975010:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 442 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975023:0:64903:0:(rw26.c:912:ll_write_end()) pos 1810432, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975026:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000010cf3656[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975028:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000d9d4f602 17ffffc0001001 3:0 ffff9b036598cd80 442 lru 00000080:00200000:1.0:1769074822.975033:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000bf7075e9 442: 1< 0 - - > 2< 1810432 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975035:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000010cf3656 00000080:00200000:1.0:1769074822.975036:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 187. 00000080:00200000:1.0:1769074822.975038:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 443 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975054:0:64903:0:(rw26.c:912:ll_write_end()) pos 1814528, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975058:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000b1d88898[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975060:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000038f820e5 17ffffc0001001 3:0 ffff9b036598ce58 443 lru 00000080:00200000:1.0:1769074822.975065:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000001709e071 443: 1< 0 - - > 2< 1814528 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975067:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000b1d88898 00000080:00200000:1.0:1769074822.975082:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 188. 00000080:00200000:1.0:1769074822.975085:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 444 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975102:0:64903:0:(rw26.c:912:ll_write_end()) pos 1818624, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975106:0:64903:0:(rw26.c:969:ll_write_end()) page@000000003e77eb42[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975108:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000007eb4636e 17ffffc0001001 3:0 ffff9b036598c438 444 lru 00000080:00200000:1.0:1769074822.975113:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000002358f4aa 444: 1< 0 - - > 2< 1818624 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975115:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000003e77eb42 00000080:00200000:1.0:1769074822.975116:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 189. 00000080:00200000:1.0:1769074822.975118:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 445 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975135:0:64903:0:(rw26.c:912:ll_write_end()) pos 1822720, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975139:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000046f08216[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975142:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000146b5a3b 17ffffc0001021 2:0 ffff9b036598c510 445 lru 00000080:00200000:1.0:1769074822.975148:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000002bcb9ba 445: 1< 0 - - > 2< 1822720 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975150:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000046f08216 00000080:00200000:1.0:1769074822.975151:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 190. 00000080:00200000:1.0:1769074822.975153:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 446 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975168:0:64903:0:(rw26.c:912:ll_write_end()) pos 1826816, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975172:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000e71059a8[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975175:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000042b2cc44 17ffffc0001001 3:0 ffff9b03432f2438 446 lru 00000080:00200000:1.0:1769074822.975180:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000009b6ec5ab 446: 1< 0 - - > 2< 1826816 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975182:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000e71059a8 00000080:00200000:1.0:1769074822.975183:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 191. 00000080:00200000:1.0:1769074822.975185:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 447 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975199:0:64903:0:(rw26.c:912:ll_write_end()) pos 1830912, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975203:0:64903:0:(rw26.c:969:ll_write_end()) page@000000006ae8d4f9[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975206:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000bc38df0f 17ffffc0001001 3:0 ffff9b03432f2510 447 lru 00000080:00200000:1.0:1769074822.975212:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000cf21897d 447: 1< 0 - - > 2< 1830912 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975213:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000006ae8d4f9 00000080:00200000:1.0:1769074822.975214:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 192. 00000080:00200000:1.0:1769074822.975216:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 448 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975278:0:64903:0:(rw26.c:912:ll_write_end()) pos 1835008, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975283:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000f409ec13[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975285:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000085177411 17ffffc0001001 3:0 ffff9b03432f2948 448 lru 00000080:00200000:1.0:1769074822.975290:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000edac421e 448: 1< 0 - - > 2< 1835008 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975292:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000f409ec13 00000080:00200000:1.0:1769074822.975293:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 193. 00000080:00200000:1.0:1769074822.975295:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 449 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975310:0:64903:0:(rw26.c:912:ll_write_end()) pos 1839104, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975314:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000ee7dd2e9[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975317:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000fc334f46 17ffffc0001001 3:0 ffff9b03432f2000 449 lru 00000080:00200000:1.0:1769074822.975322:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000fc0543d7 449: 1< 0 - - > 2< 1839104 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975324:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000ee7dd2e9 00000080:00200000:1.0:1769074822.975325:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 194. 00000080:00200000:1.0:1769074822.975327:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 450 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975346:0:64903:0:(rw26.c:912:ll_write_end()) pos 1843200, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975349:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000e844e5e4[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975352:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ae0013f5 17ffffc0001001 3:0 ffff9b03432f2ca8 450 lru 00000080:00200000:1.0:1769074822.975357:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000ef75f38f 450: 1< 0 - - > 2< 1843200 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975359:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000e844e5e4 00000080:00200000:1.0:1769074822.975360:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 195. 00000080:00200000:1.0:1769074822.975362:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 451 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975375:0:64903:0:(rw26.c:912:ll_write_end()) pos 1847296, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975379:0:64903:0:(rw26.c:969:ll_write_end()) page@000000004644ae6a[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975381:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000b33dd6c9 17ffffc0001001 3:0 ffff9b03432f25e8 451 lru 00000080:00200000:1.0:1769074822.975386:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000000b8ba852 451: 1< 0 - - > 2< 1847296 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975388:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000004644ae6a 00000080:00200000:1.0:1769074822.975390:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 196. 00000080:00200000:1.0:1769074822.975391:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 452 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975406:0:64903:0:(rw26.c:912:ll_write_end()) pos 1851392, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975409:0:64903:0:(rw26.c:969:ll_write_end()) page@000000005ebb5939[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975411:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000066d94524 17ffffc0001001 3:0 ffff9b03432f20d8 452 lru 00000080:00200000:1.0:1769074822.975416:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006c435279 452: 1< 0 - - > 2< 1851392 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975418:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000005ebb5939 00000080:00200000:1.0:1769074822.975419:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 197. 00000080:00200000:1.0:1769074822.975420:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 453 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975434:0:64903:0:(rw26.c:912:ll_write_end()) pos 1855488, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975438:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000b6ca60d8[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975440:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000f2d92f1d 17ffffc0001001 3:0 ffff9b03432f2288 453 lru 00000080:00200000:1.0:1769074822.975444:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000081df7bbe 453: 1< 0 - - > 2< 1855488 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975446:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000b6ca60d8 00000080:00200000:1.0:1769074822.975447:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 198. 00000080:00200000:1.0:1769074822.975450:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 454 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975461:0:64903:0:(rw26.c:912:ll_write_end()) pos 1859584, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975465:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000209e6113[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975467:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000f0d9ffc3 17ffffc0001001 3:0 ffff9b03432f2870 454 lru 00000080:00200000:1.0:1769074822.975471:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000002ef5d7c9 454: 1< 0 - - > 2< 1859584 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975473:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000209e6113 00000080:00200000:1.0:1769074822.975474:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 199. 00000080:00200000:1.0:1769074822.975475:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 455 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975487:0:64903:0:(rw26.c:912:ll_write_end()) pos 1863680, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975490:0:64903:0:(rw26.c:969:ll_write_end()) page@000000004993c323[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975493:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000002835d598 17ffffc0001001 3:0 ffff9b03432f2bd0 455 lru 00000080:00200000:1.0:1769074822.975498:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000014ab5bc6 455: 1< 0 - - > 2< 1863680 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975500:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000004993c323 00000080:00200000:1.0:1769074822.975501:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 200. 00000080:00200000:1.0:1769074822.975502:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 456 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975514:0:64903:0:(rw26.c:912:ll_write_end()) pos 1867776, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975518:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000037012956[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975520:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000033243211 17ffffc0001001 3:0 ffff9b03432f26c0 456 lru 00000080:00200000:1.0:1769074822.975524:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000001e75f6d9 456: 1< 0 - - > 2< 1867776 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975526:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000037012956 00000080:00200000:1.0:1769074822.975527:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 201. 00000080:00200000:1.0:1769074822.975529:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 457 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975541:0:64903:0:(rw26.c:912:ll_write_end()) pos 1871872, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975544:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000e9af20b7[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975546:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000002957b4b4 17ffffc0001001 3:0 ffff9b03432f2360 457 lru 00000080:00200000:1.0:1769074822.975551:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000005da6776 457: 1< 0 - - > 2< 1871872 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975553:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000e9af20b7 00000080:00200000:1.0:1769074822.975554:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 202. 00000080:00200000:1.0:1769074822.975556:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 458 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975571:0:64903:0:(rw26.c:912:ll_write_end()) pos 1875968, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975575:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000091a9fc5f[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975577:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000004d6d08d4 17ffffc0001001 3:0 ffff9b03432f2a20 458 lru 00000080:00200000:1.0:1769074822.975582:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000078bfac9d 458: 1< 0 - - > 2< 1875968 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975584:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000091a9fc5f 00000080:00200000:1.0:1769074822.975585:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 203. 00000080:00200000:1.0:1769074822.975587:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 459 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975600:0:64903:0:(rw26.c:912:ll_write_end()) pos 1880064, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975604:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000096a6b990[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975607:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000c29e1d46 17ffffc0001001 3:0 ffff9b03432f21b0 459 lru 00000080:00200000:1.0:1769074822.975612:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000440c8845 459: 1< 0 - - > 2< 1880064 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975614:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000096a6b990 00000080:00200000:1.0:1769074822.975615:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 204. 00000080:00200000:1.0:1769074822.975617:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 460 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975632:0:64903:0:(rw26.c:912:ll_write_end()) pos 1884160, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975635:0:64903:0:(rw26.c:969:ll_write_end()) page@000000003a96c7b8[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975638:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000007e2de51d 17ffffc0001021 2:0 ffff9b03432f2798 460 lru 00000080:00200000:1.0:1769074822.975643:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000cbaa5a4a 460: 1< 0 - - > 2< 1884160 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975645:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000003a96c7b8 00000080:00200000:1.0:1769074822.975646:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 205. 00000080:00200000:1.0:1769074822.975647:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 461 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975659:0:64903:0:(rw26.c:912:ll_write_end()) pos 1888256, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975663:0:64903:0:(rw26.c:969:ll_write_end()) page@000000004c958283[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975665:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000326bd46a 17ffffc0001001 3:0 ffff9b03432f2af8 461 lru 00000080:00200000:1.0:1769074822.975669:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000013fb8cdf 461: 1< 0 - - > 2< 1888256 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975671:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000004c958283 00000080:00200000:1.0:1769074822.975672:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 206. 00000080:00200000:1.0:1769074822.975673:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 462 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975686:0:64903:0:(rw26.c:912:ll_write_end()) pos 1892352, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975689:0:64903:0:(rw26.c:969:ll_write_end()) page@000000008715cc0f[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975692:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000005451c55c 17ffffc0001001 3:0 ffff9b03432f2d80 462 lru 00000080:00200000:1.0:1769074822.975697:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000da138ecf 462: 1< 0 - - > 2< 1892352 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975699:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000008715cc0f 00000080:00200000:1.0:1769074822.975700:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 207. 00000080:00200000:1.0:1769074822.975702:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 463 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975716:0:64903:0:(rw26.c:912:ll_write_end()) pos 1896448, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975719:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000004ed37d5[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975722:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000f34c5cf8 17ffffc0001001 3:0 ffff9b03432f2e58 463 lru 00000080:00200000:1.0:1769074822.975727:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000d64784bd 463: 1< 0 - - > 2< 1896448 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975729:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000004ed37d5 00000080:00200000:1.0:1769074822.975730:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 208. 00000080:00200000:1.0:1769074822.975732:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 464 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975746:0:64903:0:(rw26.c:912:ll_write_end()) pos 1900544, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975750:0:64903:0:(rw26.c:969:ll_write_end()) page@000000001cd36299[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975752:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ef9ec14e 17ffffc0001001 3:0 ffff9b0348d89798 464 lru 00000080:00200000:1.0:1769074822.975758:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000257a23af 464: 1< 0 - - > 2< 1900544 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975760:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000001cd36299 00000080:00200000:1.0:1769074822.975761:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 209. 00000080:00200000:1.0:1769074822.975763:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 465 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975777:0:64903:0:(rw26.c:912:ll_write_end()) pos 1904640, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975781:0:64903:0:(rw26.c:969:ll_write_end()) page@000000000f4ccf39[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975784:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000001c0bca3a 17ffffc0001001 3:0 ffff9b0348d89af8 465 lru 00000080:00200000:1.0:1769074822.975790:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000013722502 465: 1< 0 - - > 2< 1904640 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975792:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000000f4ccf39 00000080:00200000:1.0:1769074822.975793:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 210. 00000080:00200000:1.0:1769074822.975795:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 466 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975806:0:64903:0:(rw26.c:912:ll_write_end()) pos 1908736, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975810:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000adc0fa1d[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975813:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000152afde9 17ffffc0001001 3:0 ffff9b0348d89d80 466 lru 00000080:00200000:1.0:1769074822.975817:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000005036c068 466: 1< 0 - - > 2< 1908736 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975820:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000adc0fa1d 00000080:00200000:1.0:1769074822.975821:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 211. 00000080:00200000:1.0:1769074822.975823:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 467 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975836:0:64903:0:(rw26.c:912:ll_write_end()) pos 1912832, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975840:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000017a257b8[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975843:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000e4fb9cb3 17ffffc0001001 3:0 ffff9b0348d89e58 467 lru 00000080:00200000:1.0:1769074822.975849:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000acf31e66 467: 1< 0 - - > 2< 1912832 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975851:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000017a257b8 00000080:00200000:1.0:1769074822.975852:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 212. 00000080:00200000:1.0:1769074822.975854:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 468 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975867:0:64903:0:(rw26.c:912:ll_write_end()) pos 1916928, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975871:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a7849333[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975873:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000046bec073 17ffffc0001001 3:0 ffff9b0348d89438 468 lru 00000080:00200000:1.0:1769074822.975878:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000a5e1f92b 468: 1< 0 - - > 2< 1916928 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975880:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a7849333 00000080:00200000:1.0:1769074822.975881:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 213. 00000080:00200000:1.0:1769074822.975882:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 469 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975896:0:64903:0:(rw26.c:912:ll_write_end()) pos 1921024, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975900:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000034234b1[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975903:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000003dd5b93a 17ffffc0001001 3:0 ffff9b0348d89510 469 lru 00000080:00200000:1.0:1769074822.975908:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000008043c9a2 469: 1< 0 - - > 2< 1921024 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975910:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000034234b1 00000080:00200000:1.0:1769074822.975912:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 214. 00000080:00200000:1.0:1769074822.975913:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 470 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975927:0:64903:0:(rw26.c:912:ll_write_end()) pos 1925120, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975931:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000b8299b88[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975934:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000005ee7d2c2 17ffffc0001001 3:0 ffff9b0348d89948 470 lru 00000080:00200000:1.0:1769074822.975938:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000f4d95a43 470: 1< 0 - - > 2< 1925120 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975940:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000b8299b88 00000080:00200000:1.0:1769074822.975941:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 215. 00000080:00200000:1.0:1769074822.975943:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 471 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975958:0:64903:0:(rw26.c:912:ll_write_end()) pos 1929216, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975961:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a56f1e31[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975964:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000007242cb20 17ffffc0001001 3:0 ffff9b0348d89000 471 lru 00000080:00200000:1.0:1769074822.975969:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000003a028fdf 471: 1< 0 - - > 2< 1929216 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.975972:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a56f1e31 00000080:00200000:1.0:1769074822.975973:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 216. 00000080:00200000:1.0:1769074822.975975:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 472 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.975991:0:64903:0:(rw26.c:912:ll_write_end()) pos 1933312, len 4096, copied 4096 00000080:00200000:1.0:1769074822.975995:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000092e9eb16[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.975997:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ddc858fd 17ffffc0001001 3:0 ffff9b0348d89ca8 472 lru 00000080:00200000:1.0:1769074822.976002:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000077a43aa4 472: 1< 0 - - > 2< 1933312 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976004:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000092e9eb16 00000080:00200000:1.0:1769074822.976005:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 217. 00000080:00200000:1.0:1769074822.976007:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 473 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976019:0:64903:0:(rw26.c:912:ll_write_end()) pos 1937408, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976023:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000ce54d178[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976025:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ad13d4c2 17ffffc0001001 3:0 ffff9b0348d895e8 473 lru 00000080:00200000:1.0:1769074822.976029:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e8922a1e 473: 1< 0 - - > 2< 1937408 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976031:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000ce54d178 00000080:00200000:1.0:1769074822.976032:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 218. 00000080:00200000:1.0:1769074822.976034:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 474 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976047:0:64903:0:(rw26.c:912:ll_write_end()) pos 1941504, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976050:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000086c7d0a5[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976052:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000058fff64 17ffffc0001001 3:0 ffff9b0348d890d8 474 lru 00000080:00200000:1.0:1769074822.976056:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000ff5c9cba 474: 1< 0 - - > 2< 1941504 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976058:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000086c7d0a5 00000080:00200000:1.0:1769074822.976059:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 219. 00000080:00200000:1.0:1769074822.976061:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 475 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976076:0:64903:0:(rw26.c:912:ll_write_end()) pos 1945600, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976080:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000faade880[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976082:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000c95d72ba 17ffffc0001021 2:0 ffff9b0348d89288 475 lru 00000080:00200000:1.0:1769074822.976087:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000ac9d5a47 475: 1< 0 - - > 2< 1945600 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976090:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000faade880 00000080:00200000:1.0:1769074822.976091:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 220. 00000080:00200000:1.0:1769074822.976092:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 476 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976106:0:64903:0:(rw26.c:912:ll_write_end()) pos 1949696, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976111:0:64903:0:(rw26.c:969:ll_write_end()) page@000000007488c037[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976114:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000e6e10b3e 17ffffc0001001 3:0 ffff9b0348d89870 476 lru 00000080:00200000:1.0:1769074822.976119:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000004ef31168 476: 1< 0 - - > 2< 1949696 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976121:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000007488c037 00000080:00200000:1.0:1769074822.976123:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 221. 00000080:00200000:1.0:1769074822.976124:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 477 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976137:0:64903:0:(rw26.c:912:ll_write_end()) pos 1953792, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976141:0:64903:0:(rw26.c:969:ll_write_end()) page@000000006787a4d8[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976143:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000e785fd2f 17ffffc0001001 3:0 ffff9b0348d89bd0 477 lru 00000080:00200000:1.0:1769074822.976149:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000012492a87 477: 1< 0 - - > 2< 1953792 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976151:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000006787a4d8 00000080:00200000:1.0:1769074822.976152:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 222. 00000080:00200000:1.0:1769074822.976154:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 478 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976170:0:64903:0:(rw26.c:912:ll_write_end()) pos 1957888, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976175:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000011dbde57[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976177:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000de7e9462 17ffffc0001001 3:0 ffff9b0348d896c0 478 lru 00000080:00200000:1.0:1769074822.976183:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006a6e072e 478: 1< 0 - - > 2< 1957888 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976185:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000011dbde57 00000080:00200000:1.0:1769074822.976186:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 223. 00000080:00200000:1.0:1769074822.976188:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 479 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976203:0:64903:0:(rw26.c:912:ll_write_end()) pos 1961984, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976206:0:64903:0:(rw26.c:969:ll_write_end()) page@000000009886a4dd[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976208:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000005c69ec2a 17ffffc0001001 3:0 ffff9b0348d89360 479 lru 00000080:00200000:1.0:1769074822.976213:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000003d0e0a82 479: 1< 0 - - > 2< 1961984 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976215:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000009886a4dd 00000080:00200000:1.0:1769074822.976216:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 224. 00000080:00200000:1.0:1769074822.976218:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 480 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976274:0:64903:0:(rw26.c:912:ll_write_end()) pos 1966080, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976280:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000047b94b1d[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976283:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000009387bc0 17ffffc0001001 3:0 ffff9b0348d89a20 480 lru 00000080:00200000:1.0:1769074822.976288:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000beb7dc2b 480: 1< 0 - - > 2< 1966080 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976290:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000047b94b1d 00000080:00200000:1.0:1769074822.976291:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 225. 00000080:00200000:1.0:1769074822.976293:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 481 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976308:0:64903:0:(rw26.c:912:ll_write_end()) pos 1970176, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976312:0:64903:0:(rw26.c:969:ll_write_end()) page@000000006b21b13e[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976314:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000e1e91fd6 17ffffc0001001 3:0 ffff9b0348d891b0 481 lru 00000080:00200000:1.0:1769074822.976319:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000966358de 481: 1< 0 - - > 2< 1970176 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976321:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000006b21b13e 00000080:00200000:1.0:1769074822.976323:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 226. 00000080:00200000:1.0:1769074822.976324:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 482 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976338:0:64903:0:(rw26.c:912:ll_write_end()) pos 1974272, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976342:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000067479b56[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976345:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000008253aeea 17ffffc0001001 3:0 ffff9b035d8e9360 482 lru 00000080:00200000:1.0:1769074822.976350:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000af595df8 482: 1< 0 - - > 2< 1974272 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976352:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000067479b56 00000080:00200000:1.0:1769074822.976353:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 227. 00000080:00200000:1.0:1769074822.976355:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 483 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976369:0:64903:0:(rw26.c:912:ll_write_end()) pos 1978368, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976373:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000072537494[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976376:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000cb3210fa 17ffffc0001001 3:0 ffff9b035d8e9a20 483 lru 00000080:00200000:1.0:1769074822.976381:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000d74030a3 483: 1< 0 - - > 2< 1978368 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976383:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000072537494 00000080:00200000:1.0:1769074822.976384:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 228. 00000080:00200000:1.0:1769074822.976386:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 484 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976400:0:64903:0:(rw26.c:912:ll_write_end()) pos 1982464, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976416:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000870487d0[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976419:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000176bedac 17ffffc0001001 3:0 ffff9b035d8e91b0 484 lru 00000080:00200000:1.0:1769074822.976424:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000fb679bf1 484: 1< 0 - - > 2< 1982464 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976426:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000870487d0 00000080:00200000:1.0:1769074822.976427:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 229. 00000080:00200000:1.0:1769074822.976429:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 485 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976449:0:64903:0:(rw26.c:912:ll_write_end()) pos 1986560, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976453:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a235f782[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976455:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000b74d1c71 17ffffc0001001 3:0 ffff9b035d8e9798 485 lru 00000080:00200000:1.0:1769074822.976460:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e8b17b36 485: 1< 0 - - > 2< 1986560 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976462:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a235f782 00000080:00200000:1.0:1769074822.976463:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 230. 00000080:00200000:1.0:1769074822.976465:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 486 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976479:0:64903:0:(rw26.c:912:ll_write_end()) pos 1990656, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976483:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000d5ddadbc[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976486:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000d10cdbde 17ffffc0001001 3:0 ffff9b035d8e9af8 486 lru 00000080:00200000:1.0:1769074822.976491:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000d49a13a1 486: 1< 0 - - > 2< 1990656 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976493:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000d5ddadbc 00000080:00200000:1.0:1769074822.976494:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 231. 00000080:00200000:1.0:1769074822.976496:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 487 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976509:0:64903:0:(rw26.c:912:ll_write_end()) pos 1994752, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976513:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000ee6e9b3e[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976516:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000029385da0 17ffffc0001001 3:0 ffff9b035d8e9d80 487 lru 00000080:00200000:1.0:1769074822.976521:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000048f66e45 487: 1< 0 - - > 2< 1994752 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976523:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000ee6e9b3e 00000080:00200000:1.0:1769074822.976524:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 232. 00000080:00200000:1.0:1769074822.976525:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 488 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976540:0:64903:0:(rw26.c:912:ll_write_end()) pos 1998848, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976543:0:64903:0:(rw26.c:969:ll_write_end()) page@000000000fe17e36[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976546:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000002bf72057 17ffffc0001001 3:0 ffff9b035d8e9e58 488 lru 00000080:00200000:1.0:1769074822.976550:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000ad37cac6 488: 1< 0 - - > 2< 1998848 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976552:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000000fe17e36 00000080:00200000:1.0:1769074822.976553:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 233. 00000080:00200000:1.0:1769074822.976555:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 489 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976567:0:64903:0:(rw26.c:912:ll_write_end()) pos 2002944, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976571:0:64903:0:(rw26.c:969:ll_write_end()) page@000000003f4a819d[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976574:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000e08591fb 17ffffc0001001 3:0 ffff9b035d8e9438 489 lru 00000080:00200000:1.0:1769074822.976579:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b76a8873 489: 1< 0 - - > 2< 2002944 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976581:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000003f4a819d 00000080:00200000:1.0:1769074822.976582:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 234. 00000080:00200000:1.0:1769074822.976584:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 490 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976601:0:64903:0:(rw26.c:912:ll_write_end()) pos 2007040, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976606:0:64903:0:(rw26.c:969:ll_write_end()) page@000000007c687495[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976608:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000bca10bb1 17ffffc0001021 2:0 ffff9b035d8e9510 490 lru 00000080:00200000:1.0:1769074822.976614:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000519d8fa0 490: 1< 0 - - > 2< 2007040 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976616:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000007c687495 00000080:00200000:1.0:1769074822.976617:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 235. 00000080:00200000:1.0:1769074822.976619:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 491 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976631:0:64903:0:(rw26.c:912:ll_write_end()) pos 2011136, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976634:0:64903:0:(rw26.c:969:ll_write_end()) page@000000008691738d[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976637:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000bdb8bd8a 17ffffc0001001 3:0 ffff9b035d8e9948 491 lru 00000080:00200000:1.0:1769074822.976642:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000f9f7ec44 491: 1< 0 - - > 2< 2011136 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976644:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000008691738d 00000080:00200000:1.0:1769074822.976645:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 236. 00000080:00200000:1.0:1769074822.976647:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 492 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976659:0:64903:0:(rw26.c:912:ll_write_end()) pos 2015232, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976663:0:64903:0:(rw26.c:969:ll_write_end()) page@000000008ae1dc92[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976665:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000096d03e87 17ffffc0001001 3:0 ffff9b035d8e9000 492 lru 00000080:00200000:1.0:1769074822.976670:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000003a8112db 492: 1< 0 - - > 2< 2015232 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976672:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000008ae1dc92 00000080:00200000:1.0:1769074822.976674:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 237. 00000080:00200000:1.0:1769074822.976676:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 493 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976688:0:64903:0:(rw26.c:912:ll_write_end()) pos 2019328, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976691:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000d1179005[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976694:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000f63e351e 17ffffc0001001 3:0 ffff9b035d8e9ca8 493 lru 00000080:00200000:1.0:1769074822.976700:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000000c720c8a 493: 1< 0 - - > 2< 2019328 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976702:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000d1179005 00000080:00200000:1.0:1769074822.976703:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 238. 00000080:00200000:1.0:1769074822.976705:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 494 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976715:0:64903:0:(rw26.c:912:ll_write_end()) pos 2023424, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976719:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000abd52690[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976722:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000d682e598 17ffffc0001001 3:0 ffff9b035d8e95e8 494 lru 00000080:00200000:1.0:1769074822.976728:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000478a7f15 494: 1< 0 - - > 2< 2023424 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976730:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000abd52690 00000080:00200000:1.0:1769074822.976731:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 239. 00000080:00200000:1.0:1769074822.976733:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 495 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976745:0:64903:0:(rw26.c:912:ll_write_end()) pos 2027520, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976749:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000dc06ab0a[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976752:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000005330a404 17ffffc0001001 3:0 ffff9b035d8e90d8 495 lru 00000080:00200000:1.0:1769074822.976758:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000000e101743 495: 1< 0 - - > 2< 2027520 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976760:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000dc06ab0a 00000080:00200000:1.0:1769074822.976761:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 240. 00000080:00200000:1.0:1769074822.976763:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 496 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976775:0:64903:0:(rw26.c:912:ll_write_end()) pos 2031616, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976779:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000b322c342[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976782:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000006baa759 17ffffc0001001 3:0 ffff9b035d8e9288 496 lru 00000080:00200000:1.0:1769074822.976787:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000073aa3692 496: 1< 0 - - > 2< 2031616 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976789:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000b322c342 00000080:00200000:1.0:1769074822.976790:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 241. 00000080:00200000:1.0:1769074822.976792:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 497 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976802:0:64903:0:(rw26.c:912:ll_write_end()) pos 2035712, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976806:0:64903:0:(rw26.c:969:ll_write_end()) page@000000008f4d98d5[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976809:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000003f759c5b 17ffffc0001001 3:0 ffff9b035d8e9870 497 lru 00000080:00200000:1.0:1769074822.976814:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000e86f9c0a 497: 1< 0 - - > 2< 2035712 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976816:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000008f4d98d5 00000080:00200000:1.0:1769074822.976817:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 242. 00000080:00200000:1.0:1769074822.976819:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 498 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976831:0:64903:0:(rw26.c:912:ll_write_end()) pos 2039808, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976836:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a975504c[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976838:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000090c1a44 17ffffc0001001 3:0 ffff9b035d8e9bd0 498 lru 00000080:00200000:1.0:1769074822.976843:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000deeded24 498: 1< 0 - - > 2< 2039808 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976845:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a975504c 00000080:00200000:1.0:1769074822.976847:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 243. 00000080:00200000:1.0:1769074822.976849:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 499 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976860:0:64903:0:(rw26.c:912:ll_write_end()) pos 2043904, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976864:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a107e1ba[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976866:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000444534ea 17ffffc0001001 3:0 ffff9b035d8e96c0 499 lru 00000080:00200000:1.0:1769074822.976871:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000a34d51fa 499: 1< 0 - - > 2< 2043904 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976873:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a107e1ba 00000080:00200000:1.0:1769074822.976875:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 244. 00000080:00200000:1.0:1769074822.976876:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 500 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976890:0:64903:0:(rw26.c:912:ll_write_end()) pos 2048000, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976894:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000e4634eac[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976897:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000a4922ecf 17ffffc0001001 3:0 ffff9b034535e360 500 lru 00000080:00200000:1.0:1769074822.976902:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000001e59a010 500: 1< 0 - - > 2< 2048000 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976905:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000e4634eac 00000080:00200000:1.0:1769074822.976906:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 245. 00000080:00200000:1.0:1769074822.976907:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 501 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976919:0:64903:0:(rw26.c:912:ll_write_end()) pos 2052096, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976922:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000092e910dd[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976925:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @000000006a2defd7 17ffffc0001001 3:0 ffff9b034535ea20 501 lru 00000080:00200000:1.0:1769074822.976930:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000027ce648b 501: 1< 0 - - > 2< 2052096 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976932:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000092e910dd 00000080:00200000:1.0:1769074822.976933:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 246. 00000080:00200000:1.0:1769074822.976935:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 502 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976948:0:64903:0:(rw26.c:912:ll_write_end()) pos 2056192, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976952:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000a53c5478[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976955:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000053e8868f 17ffffc0001001 3:0 ffff9b034535e1b0 502 lru 00000080:00200000:1.0:1769074822.976959:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000054c8b401 502: 1< 0 - - > 2< 2056192 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976962:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000a53c5478 00000080:00200000:1.0:1769074822.976963:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 247. 00000080:00200000:1.0:1769074822.976964:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 503 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.976977:0:64903:0:(rw26.c:912:ll_write_end()) pos 2060288, len 4096, copied 4096 00000080:00200000:1.0:1769074822.976981:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000ef549863[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.976984:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000067f9052 17ffffc0001001 3:0 ffff9b034535e798 503 lru 00000080:00200000:1.0:1769074822.976989:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000002a634cc9 503: 1< 0 - - > 2< 2060288 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.976991:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000ef549863 00000080:00200000:1.0:1769074822.976992:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 248. 00000080:00200000:1.0:1769074822.976994:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 504 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.977008:0:64903:0:(rw26.c:912:ll_write_end()) pos 2064384, len 4096, copied 4096 00000080:00200000:1.0:1769074822.977012:0:64903:0:(rw26.c:969:ll_write_end()) page@000000009cb3bc33[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.977014:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000b5209c2e 17ffffc0001001 3:0 ffff9b034535eaf8 504 lru 00000080:00200000:1.0:1769074822.977019:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000051fe35cd 504: 1< 0 - - > 2< 2064384 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.977020:0:64903:0:(rw26.c:969:ll_write_end()) end page@000000009cb3bc33 00000080:00200000:1.0:1769074822.977021:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 249. 00000080:00200000:1.0:1769074822.977023:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 505 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.977039:0:64903:0:(rw26.c:912:ll_write_end()) pos 2068480, len 4096, copied 4096 00000080:00200000:1.0:1769074822.977042:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000f34b8b27[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.977045:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ed50fd71 17ffffc0001021 2:0 ffff9b034535ed80 505 lru 00000080:00200000:1.0:1769074822.977051:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000d1d4c4fb 505: 1< 0 - - > 2< 2068480 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.977053:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000f34b8b27 00000080:00200000:1.0:1769074822.977054:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 250. 00000080:00200000:1.0:1769074822.977056:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 506 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.977070:0:64903:0:(rw26.c:912:ll_write_end()) pos 2072576, len 4096, copied 4096 00000080:00200000:1.0:1769074822.977074:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000c103f4db[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.977077:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000498aa0a2 17ffffc0001001 3:0 ffff9b034535ee58 506 lru 00000080:00200000:1.0:1769074822.977081:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b284e0fa 506: 1< 0 - - > 2< 2072576 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.977084:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000c103f4db 00000080:00200000:1.0:1769074822.977085:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 251. 00000080:00200000:1.0:1769074822.977086:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 507 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.977101:0:64903:0:(rw26.c:912:ll_write_end()) pos 2076672, len 4096, copied 4096 00000080:00200000:1.0:1769074822.977104:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000013f68126[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.977107:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000078b48f97 17ffffc0001001 3:0 ffff9b034535e438 507 lru 00000080:00200000:1.0:1769074822.977112:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000df8df798 507: 1< 0 - - > 2< 2076672 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.977114:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000013f68126 00000080:00200000:1.0:1769074822.977115:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 252. 00000080:00200000:1.0:1769074822.977118:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 508 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.977133:0:64903:0:(rw26.c:912:ll_write_end()) pos 2080768, len 4096, copied 4096 00000080:00200000:1.0:1769074822.977137:0:64903:0:(rw26.c:969:ll_write_end()) page@0000000017095e87[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.977140:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000a52c4b53 17ffffc0001001 3:0 ffff9b034535e510 508 lru 00000080:00200000:1.0:1769074822.977145:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@0000000048db9bf3 508: 1< 0 - - > 2< 2080768 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.977147:0:64903:0:(rw26.c:969:ll_write_end()) end page@0000000017095e87 00000080:00200000:1.0:1769074822.977148:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 253. 00000080:00200000:1.0:1769074822.977150:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 509 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.977163:0:64903:0:(rw26.c:912:ll_write_end()) pos 2084864, len 4096, copied 4096 00000080:00200000:1.0:1769074822.977168:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000698dfe57[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.977171:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ac631f48 17ffffc0001001 3:0 ffff9b034535e948 509 lru 00000080:00200000:1.0:1769074822.977176:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000cff34ddf 509: 1< 0 - - > 2< 2084864 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.977178:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000698dfe57 00000080:00200000:1.0:1769074822.977179:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 254. 00000080:00200000:1.0:1769074822.977181:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 510 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.977191:0:64903:0:(rw26.c:912:ll_write_end()) pos 2088960, len 4096, copied 4096 00000080:00200000:1.0:1769074822.977195:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000dd65f589[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.977198:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @0000000041b9a140 17ffffc0001001 3:0 ffff9b034535e000 510 lru 00000080:00200000:1.0:1769074822.977204:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@00000000b8ee4259 510: 1< 0 - - > 2< 2088960 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.977206:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000dd65f589 00000080:00200000:1.0:1769074822.977207:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 255. 00000080:00200000:1.0:1769074822.977209:0:64903:0:(rw26.c:729:ll_write_begin()) Writing 511 of 0 to 4096 bytes 00000080:00200000:1.0:1769074822.977219:0:64903:0:(rw26.c:912:ll_write_end()) pos 2093056, len 4096, copied 4096 00000080:00200000:1.0:1769074822.977223:0:64903:0:(rw26.c:969:ll_write_end()) page@00000000ef25eb36[3 00000000e75f8658 2 1 00000000d5f56c8b] 00000080:00200000:1.0:1769074822.977226:0:64903:0:(rw26.c:969:ll_write_end()) vmpage @00000000ab335d5d 17ffffc0001001 3:0 ffff9b034535eca8 511 lru 00000080:00200000:1.0:1769074822.977268:0:64903:0:(rw26.c:969:ll_write_end()) osc-page@000000006f6ff207 511: 1< 0 - - > 2< 2093056 0 0 0x0 0x0 | 00000000e7925d05 000000002321bc12 > 3< 0 0 > 4< 0 0 8 8437760 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:1.0:1769074822.977272:0:64903:0:(rw26.c:969:ll_write_end()) end page@00000000ef25eb36 00000080:00200000:1.0:1769074822.977273:0:64903:0:(rw26.c:969:ll_write_end()) queued page: 256. 00000080:00200000:1.0:1769074822.977277:0:64903:0:(vvp_io.c:1193:vvp_io_write_commit()) commit async pages: 256, from 0, to 4096 prio 0 00000008:00010000:1.0:1769074822.977334:0:64903:0:(osc_cache.c:814:osc_extent_find()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977363:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977372:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977380:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977388:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977396:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977404:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977411:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977419:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977427:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977440:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977447:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977455:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977463:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977471:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.977486:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.977493:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977501:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977510:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977518:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977526:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977535:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977543:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977551:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977560:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977567:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977576:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977584:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977591:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977603:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977612:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.977624:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.977629:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977637:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977644:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977652:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977660:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977668:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977677:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977685:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977692:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977700:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977707:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977714:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977722:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977730:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977737:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.977749:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.977754:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977761:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977768:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977775:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977781:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977788:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977794:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977802:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977808:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977815:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977827:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977835:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977843:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977851:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977858:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.977871:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.977877:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977885:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977893:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977902:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977911:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977919:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977927:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977935:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977944:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977951:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977958:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977965:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977972:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977985:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.977993:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.978005:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.978011:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978020:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978028:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978037:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978045:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978053:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978061:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978070:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978078:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978086:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978094:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978102:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978110:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978118:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978126:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.978139:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.978143:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978153:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978161:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978169:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978177:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978186:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978192:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978200:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978208:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978216:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.978228:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980498:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980512:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980520:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980528:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.980548:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.980558:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980567:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980575:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980583:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980591:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980598:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980605:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980612:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980619:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980627:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980636:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980644:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980652:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980664:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980672:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.980684:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.980690:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980698:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980707:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980715:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980723:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980731:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980738:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980746:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980754:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980761:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980769:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980777:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980786:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980793:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980801:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.980814:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.980818:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980827:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980836:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980844:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980852:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980860:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980868:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980877:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980884:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980892:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980906:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980914:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980922:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980930:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980938:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.980950:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.980955:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980962:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980969:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980977:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980985:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.980993:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.981000:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.981007:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.981014:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.981022:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.981029:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.981038:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.981045:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.981058:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.981066:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.982379:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.982392:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982402:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982411:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982419:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982427:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982435:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982443:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982452:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982460:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982468:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982477:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982485:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982494:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982502:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982510:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.982523:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.982528:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982537:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982545:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982554:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982562:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982571:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982579:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982588:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982596:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982604:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982618:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982626:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982634:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982641:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982649:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.982661:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.982666:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982674:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982682:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982690:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982698:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982705:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982712:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982720:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982729:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982736:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982744:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982752:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982759:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982772:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982780:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.982793:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.982798:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982806:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982813:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982821:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982829:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982839:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982846:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982854:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982862:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982871:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982878:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982885:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982892:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982900:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982907:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.982918:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.982923:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982932:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982939:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982946:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982954:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982961:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982969:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982977:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982984:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.982991:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.983003:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.983009:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.983017:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.983024:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.983033:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.983046:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.983052:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.983060:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.983068:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984474:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984483:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984492:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984500:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984507:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984514:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984524:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984531:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984538:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984545:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984558:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074822.984565:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.984583:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 15, dirtied 15 00000008:00010000:1.0:1769074822.984591:0:64903:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000097c2eca ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074822.984600:0:64903:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b03606fd4f8, count 1, dirtied 1 00000080:00200000:1.0:1769074822.984605:0:64903:0:(vvp_io.c:1216:vvp_io_write_commit()) Committed 256 pages 1048576 bytes, tot: 1048576 00000080:00200000:1.0:1769074822.984609:0:64903:0:(vvp_io.c:1390:vvp_io_write_start()) f202.sanity-flr: write bytes 0, result: 1048576 00000020:00010000:1.0:1769074822.984628:0:64903:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0343c7e128 (ffff9b03531522d8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074822.984631:0:64903:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343c7e128 (ffff9b03531522d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074822.984636:0:64903:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359274130 (ffff9b0342b7d758/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074822.984643:0:64903:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074822.984654:0:64903:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074822.984670:0:64903:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343c7e128 (ffff9b03531522d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074822.984673:0:64903:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359274130 (ffff9b03531522d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074822.984685:0:64903:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00000080:00200000:1.0:1769074822.984692:0:64903:0:(file.c:2109:ll_file_io_generic()) Range unlock [256, 511] 00000080:00200000:1.0:1769074822.984701:0:64903:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x200:0x0] ignore/verify layout 0/0, layout version 3 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074822.984705:0:64903:0:(file.c:2142:ll_file_io_generic()) f202.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:1.0:1769074822.984721:0:64903:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x200:0x0] updating i_size 2097152 i_blocks 0 00000080:00200000:1.0:1769074822.984725:0:64903:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:1.0:1769074822.984763:0:64903:0:(file.c:424:ll_file_release()) START file f202.sanity-flr:[0x200000405:0x200:0x0](ffff9b03606fd348), flags 100001 00000002:00100000:1.0:1769074822.984833:0:64903:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0352a18000 x1855007616984064/t8589944058(8589944058) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 768/848 e 0 to 0 dl 1769074877 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074822.984846:0:64903:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074822.984851:0:64903:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ce9c00 x1855007616985856/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074822.984873:0:64903:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64903:1855007616985856:192.168.204.160@tcp:35:dd.0 00000100:00100000:1.0:1769074822.985035:0:64903:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a000 going to sleep for 11 seconds 00000080:00200000:1.0:1769074823.058628:0:64905:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074823.058659:0:64905:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074823.058721:0:64905:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 64905 initiator: MDT0 00010000:00010000:1.0:1769074823.058730:0:64905:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64905 initiator: MDT0 00010000:00010000:1.0:1769074823.058740:0:64905:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 64905 initiator: MDT0 00000100:00100000:1.0:1769074823.058765:0:64905:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:64905:1855007616985984:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074823.058860:0:64905:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a480 going to sleep for 50 seconds 00010000:00010000:0.0:1769074824.596697:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074825.631724:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1769074829.803774:0:65489:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074829.803806:0:65489:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.803824:0:65489:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.803842:0:65489:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.803849:0:65489:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074829.803857:0:65489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074829.803860:0:65489:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c553 00000080:00010000:1.0:1769074829.803867:0:65489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c553 from it ffffb43641ea3b58 00010000:00010000:1.0:1769074829.803871:0:65489:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.803880:0:65489:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.803896:0:65489:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074829.803908:0:65489:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), flags=13 00000080:00200000:1.0:1769074829.803913:0:65489:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=getattr 00000002:00010000:1.0:1769074829.803924:0:65489:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f203.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.803971:0:65489:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.803978:0:65489:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.803987:0:65489:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65489 initiator: MDT0 00000100:00100000:1.0:1769074829.804012:0:65489:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65489:1855007616988544:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074829.804227:0:65489:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0353184480 going to sleep for 50 seconds 00000100:00100000:1.0:1769074829.808101:0:65489:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65489:1855007616988544:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074829.808121:0:65489:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.808132:0:65489:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.808150:0:65489:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.808158:0:65489:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.808166:0:65489:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.808172:0:65489:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0344ce0800 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:1.0:1769074829.808176:0:65489:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.808183:0:65489:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.808190:0:65489:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.808196:0:65489:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074829.808202:0:65489:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.808207:0:65489:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c56f lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65489 initiator: MDT0 00000002:00100000:1.0:1769074829.808223:0:65489:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9b0342ce8a80 x1855007616988544/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 584/584 e 0 to 0 dl 1769074884 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074829.808266:0:65489:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1769074829.808291:0:65489:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.808302:0:65489:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00010000:1.0:1769074829.808310:0:65489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c553 from it ffffb43641ea3940 00010000:00010000:1.0:1769074829.808313:0:65489:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.808319:0:65489:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.808380:0:65489:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074829.808392:0:65489:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.808398:0:65489:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.808405:0:65489:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.808411:0:65489:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074829.808416:0:65489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074829.808419:0:65489:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c553 00000080:00010000:1.0:1769074829.808423:0:65489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c553 from it ffffb43641ea3be8 00010000:00010000:1.0:1769074829.808427:0:65489:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.808437:0:65489:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.808444:0:65489:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074829.808450:0:65489:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f203.sanity-flr, flags=16392 00000080:00200008:1.0:1769074829.808472:0:65489:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074829.808477:0:65489:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074829.808483:0:65489:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.808489:0:65489:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.808497:0:65489:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.808504:0:65489:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074829.808508:0:65489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074829.808510:0:65489:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c553 00000080:00010000:1.0:1769074829.808514:0:65489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c553 from it ffffb43641ea3cb8 00010000:00010000:1.0:1769074829.808517:0:65489:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.808524:0:65489:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074829.808968:0:65489:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074829.808973:0:65489:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074829.808982:0:65489:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.808990:0:65489:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.808999:0:65489:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.809004:0:65489:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074829.809008:0:65489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074829.809011:0:65489:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c553 00000080:00010000:1.0:1769074829.809015:0:65489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c553 from it ffffb43641ea3cb8 00010000:00010000:1.0:1769074829.809018:0:65489:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.809025:0:65489:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.809118:0:65489:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074829.809127:0:65489:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.809134:0:65489:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.818457:0:65489:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.818473:0:65489:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074829.818482:0:65489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074829.818487:0:65489:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c553 00000080:00010000:1.0:1769074829.818496:0:65489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c553 from it ffffb43641ea3b58 00010000:00010000:1.0:1769074829.818500:0:65489:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.818511:0:65489:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.818528:0:65489:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074829.818537:0:65489:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f203.sanity-flr, flags=8 00000080:00200000:1.0:1769074829.818566:0:65489:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074829.818582:0:65489:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.818594:0:65489:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.818603:0:65489:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.818609:0:65489:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074829.818613:0:65489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074829.818616:0:65489:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c553 00000080:00010000:1.0:1769074829.818620:0:65489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c553 from it ffffb43641ea3bf8 00010000:00010000:1.0:1769074829.818623:0:65489:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.818630:0:65489:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.818640:0:65489:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074829.818645:0:65489:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f203.sanity-flr, flags=777 00000080:00200000:1.0:1769074829.818648:0:65489:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074829.818654:0:65489:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.818660:0:65489:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.818669:0:65489:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.818675:0:65489:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074829.818679:0:65489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074829.818681:0:65489:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c553 00000080:00010000:1.0:1769074829.818684:0:65489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c553 from it ffffb43641ea3b38 00010000:00010000:1.0:1769074829.818688:0:65489:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.818695:0:65489:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.818702:0:65489:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 3 00000080:00200000:1.0:1769074829.818707:0:65489:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), file ffff9b0343a56480, open_flags a141, mode 81b6 opened 0 00000080:00200000:1.0:1769074829.818716:0:65489:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=open|creat 00000002:00010000:1.0:1769074829.818735:0:65489:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f203.sanity-flr,[0x200000405:0x202:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0120502 00010000:00010000:1.0:1769074829.818747:0:65489:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.818795:0:65489:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.818801:0:65489:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074829.818804:0:65489:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0600/0xb7d6bf619879c553 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd54cfa expref: -99 pid: 64905 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.818839:0:65489:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1000/0xb7d6bf619879c576 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65489 initiator: MDT0 00010000:00010000:1.0:1769074829.818846:0:65489:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1000/0xb7d6bf619879c576 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65489 initiator: MDT0 00000020:00100000:1.0:1769074829.818856:0:65489:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074829.818868:0:65489:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ce8a80 x1855007616988800/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 528/67824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074829.818878:0:65489:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1000/0xb7d6bf619879c576 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65489 initiator: MDT0 00000100:00100000:1.0:1769074829.818899:0:65489:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65489:1855007616988800:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074829.821159:0:65489:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0353184480 going to sleep for 50 seconds 00000100:00100000:1.0:1769074829.849406:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074829.849496:0:39330:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074829.849514:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625218176 00000100:00100000:0.0:1769074829.849523:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625218176 00000100:00100000:0.0:1769074829.849573:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8915 00000100:00100000:0.0:1769074829.849581:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342bf1850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6513:x1855007625218176:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074829.849597:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1000/0xb7d6bf619879c576 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd54da9 expref: -99 pid: 65489 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074829.849666:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074829.849673:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342bf1850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6513:x1855007625218176:12345-192.168.204.160@tcp:104: Request processed in 93us (275us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074829.849680:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8915 00000002:00010000:0.0:1769074829.914757:0:65491:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074829.914774:0:65491:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/514 (0 0) 00000080:00200000:0.0:1769074829.914781:0:65491:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1769074829.914837:0:65491:0:(namei.c:151:ll_iget()) got inode: [0x200000405:0x202:0x0](ffff9b0355c042c8): rc = 0 00000080:00200000:0.0:1769074829.914853:0:65491:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x202:0x0]: layout version change: 4294967294 -> 2 00000080:00010000:0.0:1769074829.914858:0:65491:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x202:0x0](ffff9b0355c042c8) for lock 0xb7d6bf619879c58b 00000080:00010000:0.0:1769074829.914868:0:65491:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x202:0x0](ffff9b0355c042c8) 00000080:00200008:0.0:1769074829.914884:0:65491:0:(file.c:1001:ll_file_open()) START file f203.sanity-flr:[0x200000405:0x202:0x0](ffff9b0355c042c8), flags 154400 00000002:00100000:0.0:1769074829.914909:0:65491:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0352a18a80 x1855007616990080/t8589944067(8589944067) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 768/848 e 0 to 0 dl 1769074884 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1769074829.914953:0:65491:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c58b from it ffff9b03429f4360 00010000:00010000:0.0:1769074829.914958:0:65491:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074829.914968:0:65491:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074829.914991:0:65491:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x202:0x0](ffff9b0355c042c8), inode mode 81a4 mask 44 00000080:00200000:0.0:1769074829.915083:0:65491:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x202:0x0](ffff9b0355c042c8), inode mode 81a4 mask 4 00000080:00200000:0.0:1769074829.915089:0:65491:0:(xattr.c:725:ll_xattr_get()) VFS Op:inode=[0x200000405:0x202:0x0](ffff9b0355c042c8), xattr lov 00000080:00200008:0.0:1769074829.915114:0:65491:0:(file.c:6252:ll_getattr_dentry()) START file f203.sanity-flr:[0x200000405:0x202:0x0](ffff9b0355c042c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074829.915120:0:65491:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x202:0x0](ffff9b0355c042c8),name=f203.sanity-flr 00000002:00010000:0.0:1769074829.915151:0:65491:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x202:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074829.915162:0:65491:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074829.915173:0:65491:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074829.915180:0:65491:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074829.915186:0:65491:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x202:0x0](ffff9b0355c042c8) 00000080:00010000:0.0:1769074829.915189:0:65491:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x202:0x0](ffff9b0355c042c8) for lock 0xb7d6bf619879c58b 00000080:00010000:0.0:1769074829.915194:0:65491:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c58b from it ffffb43640d17cd8 00010000:00010000:0.0:1769074829.915197:0:65491:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074829.915204:0:65491:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074829.915222:0:65491:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x202:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:0.0:1769074829.915256:0:65491:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x202:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1769074829.915260:0:65491:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1769074829.915264:0:65491:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x202:0x0] 00000008:00010000:0.0:1769074829.915310:0:65491:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074829.915314:0:65491:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568dfbb0, osc lock ffff9b0345a6d690, flags 0x400000400001002 00010000:00010000:0.0:1769074829.915363:0:65491:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074829.915371:0:65491:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 3/1,0 mode: --/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074829.915382:0:65491:0:(ldlm_request.c:1054:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff9b0365a39c00 x1855007616990208/t0(0) o101->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:0.0:1769074829.915426:0:65491:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1769074829.915440:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074829.915455:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616990208:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074829.919125:0:2406:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074829.919160:0:2406:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 6/2,0 mode: --/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074829.919175:0:2406:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074829.919186:0:2406:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074829.919201:0:2406:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 5/2,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074829.919213:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1769074829.919262:0:2406:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 6/2,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074829.919272:0:2406:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074829.919279:0:2406:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074829.919286:0:2406:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074829.919307:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616990208:192.168.204.160@tcp:101:lfs.0 00000080:00200008:1.0:1769074829.919456:0:65491:0:(file.c:424:ll_file_release()) START file f203.sanity-flr:[0x200000405:0x202:0x0](ffff9b0355c042c8), flags 154000 00000002:00100000:1.0:1769074829.919529:0:65491:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0352a18a80 x1855007616990080/t8589944067(8589944067) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 768/848 e 0 to 0 dl 1769074884 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074829.919540:0:65491:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074829.919545:0:65491:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ceb100 x1855007616990464/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074829.919566:0:65491:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65491:1855007616990464:192.168.204.160@tcp:35:lfs.0 00000100:00100000:1.0:1769074829.919635:0:65491:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0353184900 going to sleep for 11 seconds 00000100:00100000:1.0:1769074829.921857:0:65491:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65491:1855007616990464:192.168.204.160@tcp:35:lfs.0 00000002:00100000:1.0:1769074829.921891:0:65491:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0352a18a80 x1855007616990080/t8589944067(8589944067) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 768/848 e 0 to 0 dl 1769074884 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074829.922082:0:65491:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074829.922100:0:65491:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.922117:0:65491:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861800/0xb7d6bf619879c584 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54dd3 expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.922130:0:65491:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861800/0xb7d6bf619879c584 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54dd3 expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.922156:0:65491:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074829.922161:0:65491:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074829.922166:0:65491:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c584 00000080:00010000:1.0:1769074829.922172:0:65491:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c584 from it ffffb43640d17bf8 00010000:00010000:1.0:1769074829.922176:0:65491:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861800/0xb7d6bf619879c584 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54dd3 expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.922185:0:65491:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861800/0xb7d6bf619879c584 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54dd3 expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.922200:0:65491:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074829.922207:0:65491:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f203.sanity-flr, flags=265 00000080:00200000:1.0:1769074829.922212:0:65491:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x202:0x0](ffff9b0355c042c8), inode mode 81a4 mask 44 00000080:00200008:1.0:1769074829.922220:0:65491:0:(file.c:1001:ll_file_open()) START file f203.sanity-flr:[0x200000405:0x202:0x0](ffff9b0355c042c8), flags 154400 00000002:00010000:1.0:1769074829.922288:0:65491:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x202:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240114401 00010000:00010000:1.0:1769074829.922333:0:65491:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65491 initiator: MDT0 00010000:00010000:1.0:1769074829.922345:0:65491:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65491 initiator: MDT0 00000020:00100000:1.0:1769074829.922355:0:65491:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074829.922362:0:65491:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ceb100 x1855007616990720/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074829.922370:0:65491:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65491 initiator: MDT0 00000100:00100000:1.0:1769074829.922388:0:65491:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65491:1855007616990720:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074829.922538:0:65491:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0353184900 going to sleep for 50 seconds 00000100:00100000:1.0:1769074829.925279:0:65491:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65491:1855007616990720:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074829.925312:0:65491:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 4/1,0 mode: --/CR res: [0x200000405:0x202:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e0b expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.925323:0:65491:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 4/1,0 mode: --/CR res: [0x200000405:0x202:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e0b expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.925334:0:65491:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x202:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54e0b expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.925345:0:65491:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x202:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e0b expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074829.925360:0:65491:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff9b0342ceb100 x1855007616990720/t8589944069(8589944069) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/848 e 0 to 0 dl 1769074884 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074829.925373:0:65491:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0342ceb100 x1855007616990720/t8589944069(8589944069) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/848 e 0 to 0 dl 1769074884 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074829.925387:0:65491:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 240 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x202:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e0b expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074829.925401:0:65491:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x202:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e0b expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074829.925411:0:65491:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 16 res 8589935621/514 (0 0) 00000080:00200000:1.0:1769074829.925417:0:65491:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074829.925476:0:65491:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x202:0x0]: layout version change: 2 -> 2 00000080:00010000:1.0:1769074829.925483:0:65491:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x202:0x0](ffff9b0355c042c8) for lock 0xb7d6bf619879c599 00000080:00010000:1.0:1769074829.925489:0:65491:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c599 from it ffffb43640d17bf0 00010000:00010000:1.0:1769074829.925493:0:65491:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x202:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54e0b expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074829.925502:0:65491:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x202:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54e0b expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00000002:00100000:1.0:1769074829.925528:0:65491:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0342ceb100 x1855007616990720/t8589944069(8589944069) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 760/848 e 0 to 0 dl 1769074884 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074829.925626:0:65491:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x202:0x0](ffff9b0355c042c8), inode mode 81a4 mask 4 00000080:00200000:1.0:1769074829.925635:0:65491:0:(xattr.c:725:ll_xattr_get()) VFS Op:inode=[0x200000405:0x202:0x0](ffff9b0355c042c8), xattr lov 00000080:00200008:1.0:1769074829.925656:0:65491:0:(file.c:6252:ll_getattr_dentry()) START file f203.sanity-flr:[0x200000405:0x202:0x0](ffff9b0355c042c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074829.925663:0:65491:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x202:0x0](ffff9b0355c042c8),name=f203.sanity-flr 00000002:00010000:1.0:1769074829.925682:0:65491:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x202:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.925693:0:65491:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074829.925708:0:65491:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074829.925716:0:65491:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074829.925721:0:65491:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x202:0x0](ffff9b0355c042c8) 00000080:00010000:1.0:1769074829.925724:0:65491:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x202:0x0](ffff9b0355c042c8) for lock 0xb7d6bf619879c58b 00000080:00010000:1.0:1769074829.925729:0:65491:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c58b from it ffffb43640d17cd8 00010000:00010000:1.0:1769074829.925732:0:65491:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074829.925739:0:65491:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074829.925756:0:65491:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x202:0x0] ignore/verify layout 0/1, layout version 3 restore needed 0 00020000:00200000:1.0:1769074829.925769:0:65491:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x202:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074829.925772:0:65491:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1769074829.925776:0:65491:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x202:0x0] 00000008:00010000:1.0:1769074829.925798:0:65491:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074829.925801:0:65491:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360af6cb0, osc lock ffff9b0345a08000, flags 0x400000400001002 00010000:00010000:1.0:1769074829.925817:0:65491:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074829.925827:0:65491:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 2/1,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074829.925839:0:65491:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074829.925843:0:65491:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 5/1,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074829.925851:0:65491:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 4/0,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074829.925860:0:65491:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074829.925869:0:65491:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x202:0x0] updating i_size 0 i_blocks 0 00000020:00010000:1.0:1769074829.925874:0:65491:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314fd98 (ffff9b03531522d8/0) at cl_lock_release():245 00000020:00010000:1.0:1769074829.925877:0:65491:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314fd98 (ffff9b03531522d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074829.925880:0:65491:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360af6cb0 (ffff9b0342b7d758/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074829.925883:0:65491:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314fd98 (ffff9b03531522d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074829.925886:0:65491:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360af6cb0 (ffff9b03531522d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074829.925895:0:65491:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x202:0x0] ignore/verify layout 0/1, layout version 2 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074829.925923:0:65491:0:(file.c:424:ll_file_release()) START file f203.sanity-flr:[0x200000405:0x202:0x0](ffff9b0355c042c8), flags 154000 00010000:00010000:1.0:1769074829.925938:0:65491:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1a00/0xb7d6bf619879c599 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x202:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54e0b expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074829.925993:0:65491:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074829.926002:0:65491:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074829.926009:0:65491:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861800/0xb7d6bf619879c584 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54dd3 expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.926017:0:65491:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861800/0xb7d6bf619879c584 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54dd3 expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.926023:0:65491:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074829.926027:0:65491:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074829.926030:0:65491:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c584 00000080:00010000:1.0:1769074829.926034:0:65491:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c584 from it ffffb43640d17bf8 00010000:00010000:1.0:1769074829.926037:0:65491:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861800/0xb7d6bf619879c584 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54dd3 expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074829.926043:0:65491:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861800/0xb7d6bf619879c584 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54dd3 expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074829.926051:0:65491:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074829.926057:0:65491:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f203.sanity-flr, flags=265 00000080:00200000:1.0:1769074829.926062:0:65491:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x202:0x0](ffff9b0355c042c8), inode mode 81a4 mask 46 00000080:00200008:1.0:1769074829.926068:0:65491:0:(file.c:1001:ll_file_open()) START file f203.sanity-flr:[0x200000405:0x202:0x0](ffff9b0355c042c8), flags 154402 00000002:00010000:1.0:1769074829.926079:0:65491:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x202:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240114403 00010000:00010000:1.0:1769074829.926124:0:65491:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1200/0xb7d6bf619879c5a0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65491 initiator: MDT0 00010000:00010000:1.0:1769074829.926131:0:65491:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1200/0xb7d6bf619879c5a0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65491 initiator: MDT0 00000020:00100000:1.0:1769074829.926151:0:65491:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074829.926158:0:65491:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ce9880 x1855007616990976/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074829.926167:0:65491:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1200/0xb7d6bf619879c5a0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65491 initiator: MDT0 00000100:00100000:1.0:1769074829.926187:0:65491:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65491:1855007616990976:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074829.926379:0:65491:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0353184900 going to sleep for 50 seconds 00000100:00100000:0.0:1769074829.930812:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074829.932710:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625218560 00000100:00100000:1.0:1769074829.932725:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625218560 00000100:00100000:1.0:1769074829.932784:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8916 00000100:00100000:1.0:1769074829.932795:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a3aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6513:x1855007625218560:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074829.932811:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c58b lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd54def expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074829.932905:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074829.932912:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a3aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6513:x1855007625218560:12345-192.168.204.160@tcp:104: Request processed in 119us (2111us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074829.932920:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8916 00000100:00100000:1.0:1769074830.005177:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074830.005447:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625218688 00000100:00100000:0.0:1769074830.005457:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625218688 00000100:00100000:0.0:1769074830.005693:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8917 00000100:00100000:0.0:1769074830.005705:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6515:x1855007625218688:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074830.005722:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c5ae lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x202:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd54e4a expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.005802:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074830.005810:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6515:x1855007625218688:12345-192.168.204.160@tcp:104: Request processed in 108us (673us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074830.005818:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8917 00000100:00100000:1.0:1769074830.009989:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074830.010520:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625218816 00000100:00100000:0.0:1769074830.010531:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625218816 00000100:00100000:0.0:1769074830.010586:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8918 00000100:00100000:0.0:1769074830.010596:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ceb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6515:x1855007625218816:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074830.010628:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c5b5 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x203:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd54e58 expref: -99 pid: 65491 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.010738:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074830.010745:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ceb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6515:x1855007625218816:12345-192.168.204.160@tcp:104: Request processed in 151us (768us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074830.010753:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8918 00000080:00200000:1.0:1769074830.069449:0:65493:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074830.069763:0:65493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074830.069784:0:65493:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c5ca lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e9e expref: -99 pid: 65492 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074830.069801:0:65493:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c5ca lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e9e expref: -99 pid: 65492 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074830.069810:0:65493:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074830.069818:0:65493:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074830.069821:0:65493:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c5ca 00000080:00010000:1.0:1769074830.069829:0:65493:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c5ca from it ffffb43641c87bf8 00010000:00010000:1.0:1769074830.069833:0:65493:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c5ca lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54e9e expref: -99 pid: 65492 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074830.069841:0:65493:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c5ca lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e9e expref: -99 pid: 65492 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074830.069857:0:65493:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074830.069873:0:65493:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), file ffff9b03462a0180, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1769074830.069880:0:65493:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=open 00000002:00010000:1.0:1769074830.069896:0:65493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f203.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1769074830.070033:0:65493:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65493 initiator: MDT0 00010000:00010000:1.0:1769074830.070042:0:65493:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65493 initiator: MDT0 00000020:00100000:1.0:1769074830.070055:0:65493:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074830.070067:0:65493:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342cea680 x1855007616994560/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2160/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074830.070075:0:65493:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65493 initiator: MDT0 00000100:00100000:1.0:1769074830.070096:0:65493:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65493:1855007616994560:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074830.070454:0:65493:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0353184900 going to sleep for 50 seconds 00000100:00100000:1.0:1769074830.078858:0:65493:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65493:1855007616994560:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074830.078886:0:65493:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65493 initiator: MDT0 00010000:00010000:1.0:1769074830.078900:0:65493:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65493 initiator: MDT0 00010000:00010000:1.0:1769074830.078907:0:65493:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65493 initiator: MDT0 00010000:00010000:1.0:1769074830.078915:0:65493:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65493 initiator: MDT0 00010000:00010000:1.0:1769074830.078927:0:65493:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65493 initiator: MDT0 00010000:00010000:1.0:1769074830.078933:0:65493:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0344ce0800 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:1.0:1769074830.078938:0:65493:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65493 initiator: MDT0 00010000:00010000:1.0:1769074830.078946:0:65493:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65493 initiator: MDT0 00010000:00010000:1.0:1769074830.079042:0:65493:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65493 initiator: MDT0 00010000:00010000:1.0:1769074830.079050:0:65493:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074830.079056:0:65493:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65493 initiator: MDT0 00010000:00010000:1.0:1769074830.079061:0:65493:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0800/0xb7d6bf619879c5d8 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65493 initiator: MDT0 00000002:00100000:1.0:1769074830.079082:0:65493:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0342cea680 x1855007616994560/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2160/608 e 0 to 0 dl 1769074885 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074830.079096:0:65493:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1769074830.079382:0:65493:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074830.079403:0:65493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074830.079417:0:65493:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c5ca lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e9e expref: -99 pid: 65492 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074830.079428:0:65493:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c5ca lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e9e expref: -99 pid: 65492 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074830.079439:0:65493:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074830.079444:0:65493:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074830.079447:0:65493:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c5ca 00000080:00010000:1.0:1769074830.079453:0:65493:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c5ca from it ffffb43641c87bb0 00010000:00010000:1.0:1769074830.079457:0:65493:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c5ca lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54e9e expref: -99 pid: 65492 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074830.079465:0:65493:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c5ca lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54e9e expref: -99 pid: 65492 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074830.079582:0:65493:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074830.079595:0:65493:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 304000 00000080:00200080:1.0:1769074830.079694:0:65493:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=c0086915 arg=11d43d0 00000100:00100000:1.0:1769074830.079757:0:65493:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65493:1855007616994688:192.168.204.160@tcp:34:lfs.0 00000100:00100000:1.0:1769074830.079892:0:65493:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0353184900 going to sleep for 50 seconds 00000100:00100000:0.0:1769074830.138035:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074830.138081:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625219072 00000100:00100000:0.0:1769074830.138091:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625219072 00000100:00100000:0.0:1769074830.138151:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8919 00000100:00100000:0.0:1769074830.138161:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a3a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6514:x1855007625219072:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074830.138175:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861800/0xb7d6bf619879c5ed lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x204:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd54f4d expref: -99 pid: 65495 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.138271:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074830.138278:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a3a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6514:x1855007625219072:12345-192.168.204.160@tcp:104: Request processed in 120us (279us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074830.138286:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8919 00000100:00100000:0.0:1769074830.213970:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074830.214143:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625219328 00000100:00100000:1.0:1769074830.214154:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625219328 00000100:00100000:1.0:1769074830.214210:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8920 00000100:00100000:1.0:1769074830.214221:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a3b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28519:x1855007625219328:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074830.214271:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349292800/0xb7d6bf619879c592 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd54dfd expref: -99 pid: 65491 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074830.214345:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074830.214352:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a3b800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28519:x1855007625219328:12345-192.168.204.160@tcp:104: Request processed in 134us (404us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074830.214360:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8920 00000100:00100000:1.0:1769074830.220361:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616997632:192.168.204.160@tcp:16:lfs.0 00000100:00100000:0.0:1769074830.225058:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616997632:192.168.204.160@tcp:16:lfs.0 00000080:00200000:1.0:1769074830.225157:0:65495:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x204:0x0] updating i_size 0 i_blocks 0 00000080:00200000:1.0:1769074830.225176:0:65495:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x204:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1769074830.237185:0:65495:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x202:0x0](ffff9b0355c042c8) cmd=801066da arg=7fffc2acba40 00000080:00200000:1.0:1769074830.237209:0:65495:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x202:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074830.237224:0:65495:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:0.0:1769074830.237425:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616997760:192.168.204.160@tcp:1:lfs.0 00000100:00100000:0.0:1769074830.240939:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007616997760:192.168.204.160@tcp:1:lfs.0 00000080:00200008:1.0:1769074830.241227:0:65495:0:(llite_lib.c:2314:ll_setattr_raw()) START file . :VOLATILE:0000:247519EB:fd=03:[0x200000405:0x204:0x0](ffff9b0355d45b88) current size 0, valid attrs 101f0, mode 9b03, uid 0, gid 0, new size 4294967296, atime 1769074829.000000000, mtime 1769074829.000000000, ctime 1769074830.000000000 00010000:00010000:1.0:1769074830.241299:0:65495:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c602 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x204:0x0].0x0 bits 0x1a/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd54f7e expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074830.241319:0:65495:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x204:0x0]: losing layout lock 00010000:00010000:1.0:1769074830.241361:0:65495:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c602 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x204:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd54f7e expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074830.241368:0:65495:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074830.241371:0:65495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c602 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x204:0x0].0x0 bits 0x1a/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd54f7e expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074830.241395:0:65495:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074830.241403:0:65495:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342cea300 x1855007616998016/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074830.241424:0:65495:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616998016:192.168.204.160@tcp:36:lfs.0 00000100:00100000:1.0:1769074830.241489:0:65495:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0353184480 going to sleep for 50 seconds 00000100:00100000:0.0:1769074830.244882:0:65495:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616998016:192.168.204.160@tcp:36:lfs.0 00000080:00200000:0.0:1769074830.244974:0:65495:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x204:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00010000:0.0:1769074830.244998:0:65495:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000405:0x204:0x0](ffff9b0355d45b88) 00000002:00010000:0.0:1769074830.245008:0:65495:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x204:0x0]) in obj [0x200000405:0x204:0x0], intent: layout flags 0 00010000:00010000:0.0:1769074830.245058:0:65495:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 3/1,0 mode: --/CR res: [0x200000405:0x204:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65495 initiator: MDT0 00010000:00010000:0.0:1769074830.245067:0:65495:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 3/1,0 mode: --/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65495 initiator: MDT0 00010000:00010000:0.0:1769074830.245077:0:65495:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 3/1,0 mode: --/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65495 initiator: MDT0 00000100:00100000:0.0:1769074830.245098:0:65495:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616998144:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074830.245184:0:65495:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2c00 going to sleep for 50 seconds 00000100:00100000:0.0:1769074830.247878:0:65495:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616998144:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074830.247909:0:65495:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.247927:0:65495:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00000002:00100000:0.0:1769074830.247940:0:65495:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9b0365a38e00 x1855007616998144/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 376/504 e 0 to 0 dl 1769074885 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074830.247962:0:65495:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 160 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00000002:00010000:0.0:1769074830.247975:0:65495:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.247986:0:65495:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 16 res 8589935621/516 (0 0) 00000080:00200000:0.0:1769074830.247992:0:65495:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200000405:0x204:0x0] suppgids 0 0: rc 0 00000080:00010000:0.0:1769074830.248010:0:65495:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x204:0x0](ffff9b0355d45b88) for lock 0xb7d6bf619879c61e 00000080:00010000:0.0:1769074830.248017:0:65495:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c61e from it ffffb43641da7a78 00010000:00010000:0.0:1769074830.248022:0:65495:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.248030:0:65495:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.248052:0:65495:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.248065:0:65495:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00000080:00010000:0.0:1769074830.248072:0:65495:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000405:0x204:0x0](00000000d1d3ed2d) being reconfigured ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074830.248147:0:65495:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x204:0x0]: layout version change: 4294967294 -> 1 00010000:00010000:0.0:1769074830.248153:0:65495:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.248160:0:65495:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00020000:00200000:0.0:1769074830.248178:0:65495:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074830.248208:0:65495:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1769074830.248222:0:65495:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074830.248227:0:65495:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352ad27b0, osc lock ffff9b0345a6d430, flags 0x0 00000008:00010000:0.0:1769074830.248230:0:65495:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074830.248256:0:65495:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352ad27f0, osc lock ffff9b0345a6c4c0, flags 0x0 00010000:00010000:0.0:1769074830.248269:0:65495:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b034921d000 type 11 mode 2 res 10737419265/710 (0 18446744073709551615) 00010000:00010000:0.0:1769074830.248303:0:65495:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.248311:0:65495:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.248319:0:65495:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074830.248341:0:65495:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616998272:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074830.248418:0:65495:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2e40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074830.250704:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074830.250941:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625219456 00000100:00100000:0.0:1769074830.250961:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625219456 00000100:00100000:0.0:1769074830.251017:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8921 00000100:00100000:0.0:1769074830.251028:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ceaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:36516:x1855007625219456:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074830.251044:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c609 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd54f85 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.251115:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074830.251122:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ceaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:36516:x1855007625219456:12345-192.168.204.160@tcp:104: Request processed in 95us (426us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074830.251139:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8921 00000100:00100000:1.0:1769074830.254812:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074830.254997:0:39330:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074830.255016:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625219584 00000100:00100000:0.0:1769074830.255026:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625219584 00000100:00100000:0.0:1769074830.255081:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8922 00000100:00100000:0.0:1769074830.255092:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0348c2fc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8424:x1855007625219584:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074830.255106:0:39330:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.255118:0:39330:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.255141:0:39330:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.255158:0:39330:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.255170:0:39330:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.255185:0:39330:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000000000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.255202:0:39330:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b0349292000) 00000100:00100000:0.0:1769074830.255403:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0348c2fc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8424:x1855007625219584:12345-192.168.204.160@tcp:105: Request processed in 312us (599us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074830.255411:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8922 00000100:00100000:0.0:1769074830.257412:0:65495:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616998272:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074830.257444:0:65495:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.257461:0:65495:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 5/0,2 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074830.257473:0:65495:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 4/0,2 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.257496:0:65495:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 5/0,2 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.257504:0:65495:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.257536:0:65495:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0343b61800 type 11 mode 2 res 11811161089/712 (0 18446744073709551615) 00010000:00010000:0.0:1769074830.257572:0:65495:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.257581:0:65495:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.257591:0:65495:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074830.257620:0:65495:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616998528:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074830.257694:0:65495:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2e40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074830.263040:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074830.263150:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625219712 00000100:00100000:0.0:1769074830.263160:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625219712 00000100:00100000:0.0:1769074830.263214:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8923 00000100:00100000:0.0:1769074830.263225:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:36516:x1855007625219712:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074830.263277:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c610 lrc: 2/0,0 mode: PR/PR res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd54f8c expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.263351:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074830.263358:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:36516:x1855007625219712:12345-192.168.204.160@tcp:104: Request processed in 135us (327us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074830.263365:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8923 00000100:00100000:1.0:1769074830.266409:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074830.266618:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625219840 00000100:00100000:0.0:1769074830.266629:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625219840 00000100:00100000:0.0:1769074830.266683:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8924 00000100:00100000:0.0:1769074830.266692:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8424:x1855007625219840:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074830.266708:0:39329:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.266721:0:39329:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.266730:0:39329:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.266740:0:39329:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.266749:0:39329:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.266760:0:39329:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000000000 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.266778:0:39329:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b0349293a00) 00000100:00100000:0.0:1769074830.266830:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8424:x1855007625219840:12345-192.168.204.160@tcp:105: Request processed in 139us (432us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074830.266837:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8924 00000020:00010000:0.0:1769074830.271101:0:65495:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03590b07b8 (ffff9b0342b7d368/0) at cl_lock_release():245 00000020:00010000:0.0:1769074830.271106:0:65495:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03590b07b8 (ffff9b0342b7d368/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074830.271111:0:65495:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352ad27b0 (ffff9b0342b7d518/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074830.271121:0:65495:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.271149:0:65495:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074830.271163:0:65495:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352ad27f0 (ffff9b0342b7dd88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074830.271168:0:65495:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.271176:0:65495:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074830.271183:0:65495:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03590b07b8 (ffff9b0342b7d368/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074830.271186:0:65495:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352ad27b0 (ffff9b0342b7d368/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074830.271191:0:65495:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352ad27f0 (ffff9b0342b7d368/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074830.271208:0:65495:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x204:0x0] ignore/verify layout 0/0, layout version 1 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1769074830.271220:0:65495:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file . :VOLATILE:0000:247519EB:fd=03:[0x200000405:0x204:0x0](ffff9b0355d45b88) current size 0, valid attrs 101f0, mode 9b03, uid 0, gid 0, new size 4294967296, atime 1769074829.000000000, mtime 1769074829.000000000, ctime 1769074830.000000000, rc 0 00000080:00200008:0.0:1769074830.271286:0:65495:0:(llite_lib.c:2314:ll_setattr_raw()) START file f203.sanity-flr:[0x200000405:0x202:0x0](ffff9b0355c042c8) current size 0, valid attrs 101f0, mode b436, uid 1157399296, gid -25853, new size 140736459487776, atime 1769074829.000000000, mtime 1769074829.000000000, ctime 1769074830.000000000 00010000:00010000:0.0:1769074830.271317:0:65495:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860e00/0xb7d6bf619879c5e6 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd54f38 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.271338:0:65495:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c5c3 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x202:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd54e89 expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.271383:0:65495:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860e00/0xb7d6bf619879c5e6 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd54f38 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.271390:0:65495:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074830.271393:0:65495:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860e00/0xb7d6bf619879c5e6 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x202:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd54f38 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00000020:00100000:0.0:1769074830.271412:0:65495:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074830.271420:0:65495:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3aa00 x1855007616999168/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074830.271441:0:65495:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616999168:192.168.204.160@tcp:36:lfs.0 00000100:00100000:0.0:1769074830.271523:0:65495:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2780 going to sleep for 50 seconds 00000100:00100000:0.0:1769074830.275185:0:65495:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616999168:192.168.204.160@tcp:36:lfs.0 00000080:00200000:0.0:1769074830.275258:0:65495:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x202:0x0] ignore/verify layout 0/0, layout version 1 restore needed 0 00020000:00200000:0.0:1769074830.275278:0:65495:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1769074830.275295:0:65495:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:0.0:1769074830.275312:0:65495:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1769074830.275316:0:65495:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568dffb0, osc lock ffff9b0345a6c4c0, flags 0x0 00010000:00010000:0.0:1769074830.275336:0:65495:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b034921d000 type 11 mode 2 res 10737419265/709 (0 18446744073709551615) 00010000:00010000:0.0:1769074830.275381:0:65495:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.275391:0:65495:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.275401:0:65495:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074830.275419:0:65495:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616999296:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074830.275601:0:65495:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2780 going to sleep for 11 seconds 00000100:00100000:1.0:1769074830.280528:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074830.280642:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625219968 00000100:00100000:0.0:1769074830.280654:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625219968 00000100:00100000:0.0:1769074830.280719:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8925 00000100:00100000:0.0:1769074830.280729:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ceb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28955:x1855007625219968:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074830.280746:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861800/0xb7d6bf619879c5fb lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd54f70 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.280827:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074830.280834:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ceb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28955:x1855007625219968:12345-192.168.204.160@tcp:104: Request processed in 107us (318us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074830.280842:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8925 00000100:00100000:1.0:1769074830.285017:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074830.285058:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625220096 00000100:00100000:1.0:1769074830.285069:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625220096 00000100:00100000:1.0:1769074830.285122:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8926 00000100:00100000:1.0:1769074830.285142:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ceaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8424:x1855007625220096:12345-192.168.204.160@tcp:105: 00010000:00010000:1.0:1769074830.285158:0:39329:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074830.285170:0:39329:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074830.285179:0:39329:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074830.285190:0:39329:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074830.285199:0:39329:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074830.285208:0:39329:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000000000 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074830.285227:0:39329:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b0349292400) 00000100:00100000:1.0:1769074830.285330:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ceaa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8424:x1855007625220096:12345-192.168.204.160@tcp:105: Request processed in 199us (324us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074830.285339:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8926 00000100:00100000:0.0:1769074830.286059:0:65495:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65495:1855007616999296:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074830.286080:0:65495:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.286096:0:65495:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 5/0,2 mode: PW/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000000020000 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074830.286108:0:65495:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 4/0,2 mode: PW/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020000020000 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.286121:0:65495:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 5/0,2 mode: PW/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074830.286140:0:65495:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074830.286261:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616999552:192.168.204.160@tcp:2:lfs.0 00000100:00100000:0.0:1769074830.289857:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007616999552:192.168.204.160@tcp:2:lfs.0 00000100:00100000:1.0:1769074830.293720:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074830.293825:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625220224 00000100:00100000:0.0:1769074830.293836:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625220224 00000100:00100000:0.0:1769074830.293890:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8927 00000100:00100000:0.0:1769074830.293899:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6515:x1855007625220224:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074830.293916:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c5c3 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x202:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd54e89 expref: -99 pid: 65491 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.293994:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074830.294001:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6515:x1855007625220224:12345-192.168.204.160@tcp:104: Request processed in 103us (294us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074830.294009:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8927 00000100:00100000:1.0:1769074830.300391:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074830.300526:0:39329:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074830.300545:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625220352 00000100:00100000:0.0:1769074830.300554:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625220352 00000100:00100000:0.0:1769074830.300600:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8928 00000100:00100000:0.0:1769074830.300609:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0348c2cc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6515:x1855007625220352:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074830.300624:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c61e lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x204:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd54fc4 expref: -99 pid: 65495 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074830.300700:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074830.300707:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0348c2cc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6515:x1855007625220352:12345-192.168.204.160@tcp:104: Request processed in 100us (326us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074830.300717:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8928 00000080:00200000:0.0:1769074830.371171:0:65496:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074830.371202:0:65496:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074830.371375:0:65496:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.371386:0:65496:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.371396:0:65496:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 65496 initiator: MDT0 00000100:00100000:0.0:1769074830.371423:0:65496:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65496:1855007617000832:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074830.371662:0:65496:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2900 going to sleep for 50 seconds 00000100:00100000:0.0:1769074830.375587:0:65496:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65496:1855007617000832:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074830.375667:0:65496:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.375680:0:65496:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.375690:0:65496:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.375701:0:65496:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.375711:0:65496:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.375725:0:65496:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.375733:0:65496:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074830.375744:0:65496:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b0365a3b800 x1855007617000832/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 576/688 e 0 to 0 dl 1769074885 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074830.375759:0:65496:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.375772:0:65496:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:0.0:1769074830.375779:0:65496:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074830.375807:0:65496:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:0.0:1769074830.375813:0:65496:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c641 00000080:00010000:0.0:1769074830.375819:0:65496:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074830.375822:0:65496:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c641 from it ffffb43641453bf8 00010000:00010000:0.0:1769074830.375825:0:65496:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.375833:0:65496:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074830.375857:0:65496:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074830.375876:0:65496:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), file ffff9b0344c97080, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1769074830.375883:0:65496:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=open 00000002:00010000:0.0:1769074830.375907:0:65496:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f203.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:0.0:1769074830.376044:0:65496:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.376052:0:65496:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65496 initiator: MDT0 00000020:00100000:0.0:1769074830.376063:0:65496:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074830.376070:0:65496:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3b800 x1855007617001088/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2160/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074830.376078:0:65496:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65496 initiator: MDT0 00000100:00100000:0.0:1769074830.376099:0:65496:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65496:1855007617001088:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074830.376431:0:65496:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2900 going to sleep for 50 seconds 00000100:00100000:0.0:1769074830.380880:0:65496:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65496:1855007617001088:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074830.380903:0:65496:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.380915:0:65496:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.380921:0:65496:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.380928:0:65496:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.380936:0:65496:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.380941:0:65496:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0349293e00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074830.381025:0:65496:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.381032:0:65496:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.381040:0:65496:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.381046:0:65496:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074830.381052:0:65496:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65496 initiator: MDT0 00010000:00010000:0.0:1769074830.381058:0:65496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293e00/0xb7d6bf619879c648 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 65496 initiator: MDT0 00000002:00100000:0.0:1769074830.381077:0:65496:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0365a3b800 x1855007617001088/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2160/608 e 0 to 0 dl 1769074885 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074830.381092:0:65496:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1769074830.381317:0:65496:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074830.381341:0:65496:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074830.381356:0:65496:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.381366:0:65496:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074830.381372:0:65496:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074830.381377:0:65496:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074830.381380:0:65496:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c641 00000080:00010000:0.0:1769074830.381385:0:65496:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c641 from it ffffb43641453bb0 00010000:00010000:0.0:1769074830.381389:0:65496:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074830.381396:0:65496:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074830.381410:0:65496:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074830.381421:0:65496:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 304000 00000080:00200080:0.0:1769074830.381454:0:65496:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=c0086915 arg=138f3d0 00000100:00100000:0.0:1769074830.381679:0:65496:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65496:1855007617001216:192.168.204.160@tcp:34:lfs.0 00000100:00100000:0.0:1769074830.381816:0:65496:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352ad2900 going to sleep for 50 seconds 00000100:00100000:0.0:1769074830.386635:0:65496:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65496:1855007617001216:192.168.204.160@tcp:34:lfs.0 00000080:00200008:0.0:1769074830.386892:0:65496:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 304000 00000080:00200000:1.0:1769074830.426823:0:65498:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074830.426857:0:65498:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074830.426877:0:65498:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074830.426891:0:65498:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074830.426898:0:65498:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074830.426906:0:65498:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074830.426909:0:65498:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c641 00000080:00010000:1.0:1769074830.426915:0:65498:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c641 from it ffffb436420cbbf8 00010000:00010000:1.0:1769074830.426918:0:65498:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074830.426925:0:65498:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074830.426937:0:65498:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074830.427079:0:65498:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), file ffff9b0351f71680, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1769074830.427086:0:65498:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f203.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=open 00000002:00010000:1.0:1769074830.427100:0:65498:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f203.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0304001 00010000:00010000:1.0:1769074830.427162:0:65498:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1800/0xb7d6bf619879c656 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 65498 initiator: MDT0 00010000:00010000:1.0:1769074830.427172:0:65498:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1800/0xb7d6bf619879c656 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65498 initiator: MDT0 00000020:00100000:1.0:1769074830.427182:0:65498:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074830.427194:0:65498:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ce8a80 x1855007617001856/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2160/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074830.427202:0:65498:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0344ce1800/0xb7d6bf619879c656 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 65498 initiator: MDT0 00000100:00100000:1.0:1769074830.427222:0:65498:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:65498:1855007617001856:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074830.427614:0:65498:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0353184d80 going to sleep for 50 seconds 00010000:00010000:0.0:1769074834.836715:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074836.891131:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074837.922012:0:66063:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204a: FLR write/stale/resync tests with self-extending mirror ========================================================== 04:40:36 (1769074836) 00000100:00100000:0.0:1769074838.158225:0:66083:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66083:1855007617003264:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074838.158262:0:66083:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c664 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66083 initiator: MDT0 00010000:00010000:0.0:1769074838.158276:0:66083:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c664 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66083 initiator: MDT0 00010000:00010000:0.0:1769074838.158283:0:66083:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c664 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66083 initiator: MDT0 00010000:00010000:0.0:1769074838.158290:0:66083:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c664 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66083 initiator: MDT0 00010000:00010000:0.0:1769074838.158297:0:66083:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c664 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66083 initiator: MDT0 00010000:00010000:0.0:1769074838.158303:0:66083:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860200 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074838.158308:0:66083:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c664 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 66083 initiator: MDT0 00010000:00010000:0.0:1769074838.158314:0:66083:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c664 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 66083 initiator: MDT0 00010000:00010000:0.0:1769074838.158321:0:66083:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c664 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 66083 initiator: MDT0 00010000:00010000:0.0:1769074838.158327:0:66083:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074838.158333:0:66083:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c664 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66083 initiator: MDT0 00010000:00010000:0.0:1769074838.158339:0:66083:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c664 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66083 initiator: MDT0 00000002:00100000:0.0:1769074838.158358:0:66083:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9b03482c5500 x1855007617003264/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074893 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074838.158372:0:66083:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1769074838.158398:0:66083:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.158411:0:66083:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00010000:0.0:1769074838.158420:0:66083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c641 from it ffffb43640e33940 00010000:00010000:0.0:1769074838.158424:0:66083:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.158432:0:66083:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.158517:0:66083:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074838.158533:0:66083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074838.158540:0:66083:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.158549:0:66083:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.158555:0:66083:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074838.158561:0:66083:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074838.158564:0:66083:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c641 00000080:00010000:0.0:1769074838.158568:0:66083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c641 from it ffffb43640e33be8 00010000:00010000:0.0:1769074838.158572:0:66083:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.158579:0:66083:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.158587:0:66083:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074838.158594:0:66083:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=16392 00000080:00200008:0.0:1769074838.158615:0:66083:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074838.158619:0:66083:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074838.158624:0:66083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074838.158629:0:66083:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.158637:0:66083:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.158643:0:66083:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074838.158647:0:66083:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074838.158649:0:66083:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c641 00000080:00010000:0.0:1769074838.158652:0:66083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c641 from it ffffb43640e33cb8 00010000:00010000:0.0:1769074838.158655:0:66083:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.158661:0:66083:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200008:0.0:1769074838.159194:0:66083:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074838.159199:0:66083:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074838.159207:0:66083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074838.159214:0:66083:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.159222:0:66083:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.159228:0:66083:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074838.159258:0:66083:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074838.159261:0:66083:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c641 00000080:00010000:0.0:1769074838.159266:0:66083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c641 from it ffffb43640e33cb8 00010000:00010000:0.0:1769074838.159269:0:66083:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.159276:0:66083:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.159390:0:66083:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074838.159399:0:66083:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074838.159406:0:66083:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.159414:0:66083:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.159421:0:66083:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074838.159425:0:66083:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074838.159427:0:66083:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c641 00000080:00010000:0.0:1769074838.159431:0:66083:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c641 from it ffffb43640e33bb0 00010000:00010000:0.0:1769074838.159434:0:66083:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.159440:0:66083:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.159448:0:66083:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074838.159455:0:66083:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:0.0:1769074838.159474:0:66083:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=c03066f0 arg=7fff583af060 00000080:00200000:0.0:1769074838.159486:0:66083:0:(dir.c:483:ll_dir_setdirstripe()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) name=d204a.sanity-flr stripe_offset=0 stripe_count=2, hash_type=2 00010000:00010000:0.0:1769074838.159510:0:66083:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.159564:0:66083:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.159575:0:66083:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074838.159578:0:66083:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c641 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd5500a expref: -99 pid: 65496 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074838.159598:0:66083:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074838.159606:0:66083:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3b800 x1855007617003520/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 568/2080 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074838.159627:0:66083:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66083:1855007617003520:192.168.204.160@tcp:36:lfs.0 00000100:00100000:0.0:1769074838.159716:0:66083:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ac00 going to sleep for 50 seconds 00000080:00200000:0.0:1769074838.216757:0:66089:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074838.216799:0:66089:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074838.216872:0:66089:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66089 initiator: MDT0 00010000:00010000:0.0:1769074838.216883:0:66089:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66089 initiator: MDT0 00010000:00010000:0.0:1769074838.216901:0:66089:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66089 initiator: MDT0 00000100:00100000:0.0:1769074838.216932:0:66089:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66089:1855007617003648:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074838.217064:0:66089:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a0c0 going to sleep for 50 seconds 00000080:00200000:0.0:1769074838.265829:0:66089:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x240000404:0x4b:0x0](ffff9b03606fa1c8) 00000080:00200000:0.0:1769074838.265839:0:66089:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x240000404:0x4b:0x0], updating i_size 0 00000080:00200000:0.0:1769074838.265874:0:66089:0:(file.c:6809:ll_layout_conf()) [0x240000404:0x4b:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1769074838.265880:0:66089:0:(namei.c:1535:ll_create_node()) setting l_ast_data to inode [0x240000404:0x4b:0x0](ffff9b03606fa1c8) 00000080:00010000:0.0:1769074838.265883:0:66089:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4b:0x0](ffff9b03606fa1c8) for lock 0xb7d6bf619879c695 00000080:00010000:0.0:1769074838.265893:0:66089:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4b:0x0](ffff9b03606fa1c8) 00000080:00200008:0.0:1769074838.265910:0:66089:0:(file.c:1001:ll_file_open()) START file f204a.sanity-flr:[0x240000404:0x4b:0x0](ffff9b03606fa1c8), flags 120501 00000002:00100000:0.0:1769074838.265934:0:66089:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0352a19180 x1855007617004544/t4294968351(4294968351) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 536/608 e 0 to 0 dl 1769074893 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1769074838.265971:0:66089:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c695 from it ffff9b03429f4ba0 00010000:00010000:0.0:1769074838.265976:0:66089:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c695 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55176 expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.265985:0:66089:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c695 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55176 expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.266016:0:66089:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x240000404:0x4b:0x0](ffff9b03606fa1c8), inode mode 81a4 mask 40 00000080:00200000:0.0:1769074838.266194:0:66089:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x240000404:0x4b:0x0](ffff9b03606fa1c8), inode mode 81a4 mask 2 00000080:00200000:0.0:1769074838.266204:0:66089:0:(xattr.c:406:ll_xattr_set()) VFS Op:inode=[0x240000404:0x4b:0x0](ffff9b03606fa1c8), xattr lov 00000002:00010000:0.0:1769074838.266222:0:66089:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4b:0x0]) in obj [0x240000404:0x4b:0x0], intent: open flags 040000002 00010000:00010000:0.0:1769074838.266688:0:66089:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c69c lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4b:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66089 initiator: MDT0 00010000:00010000:0.0:1769074838.266700:0:66089:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c69c lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4b:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66089 initiator: MDT0 00000020:00100000:0.0:1769074838.266713:0:66089:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074838.266721:0:66089:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a38000 x1855007617004800/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2216/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074838.266730:0:66089:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0349292200/0xb7d6bf619879c69c lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4b:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66089 initiator: MDT0 00000100:00100000:0.0:1769074838.266754:0:66089:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66089:1855007617004800:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074838.266959:0:66089:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ab40 going to sleep for 50 seconds 00000100:00100000:0.0:1769074838.285277:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074838.285658:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625226368 00000100:00100000:1.0:1769074838.285670:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625226368 00000100:00100000:1.0:1769074838.285728:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8929 00000100:00100000:1.0:1769074838.285738:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a38700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:10283:x1855007625226368:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074838.285755:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c695 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd55176 expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074838.285840:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074838.285847:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a38700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:10283:x1855007625226368:12345-192.168.204.160@tcp:104: Request processed in 111us (582us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074838.285854:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8929 00000080:00200000:1.0:1769074838.306848:0:66090:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074838.306876:0:66090:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074838.306895:0:66090:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074838.306910:0:66090:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074838.306918:0:66090:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074838.306926:0:66090:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074838.306929:0:66090:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:1.0:1769074838.306936:0:66090:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb4364155bbf8 00010000:00010000:1.0:1769074838.306939:0:66090:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074838.306947:0:66090:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074838.306963:0:66090:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074838.306971:0:66090:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=25 00000080:00200000:1.0:1769074838.306978:0:66090:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x205:0x0](ffff9b0355d47448), inode mode 41ed mask 1 00000080:00200000:1.0:1769074838.306990:0:66090:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x205:0x0](ffff9b0355d47448), inode mode 41ed mask 3 00000080:00200000:1.0:1769074838.307008:0:66090:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000405:0x205:0x0](ffff9b0355d47448), file ffff9b036085a480, open_flags 8041, mode 81b6 opened 0 00000080:00200000:1.0:1769074838.307019:0:66090:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000405:0x205:0x0](ffff9b0355d47448), intent=open|creat 00000002:00010000:1.0:1769074838.307045:0:66090:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204a.sanity-flr,[0x240000404:0x4c:0x0]) in obj [0x240000405:0x7:0x0], intent: open|creat flags 0100102 00010000:00010000:1.0:1769074838.307104:0:66090:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0e00/0xb7d6bf619879c6a3 lrc: 3/0,1 mode: --/CW res: [0x240000405:0x7:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66090 initiator: MDT0 00010000:00010000:1.0:1769074838.307112:0:66090:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0e00/0xb7d6bf619879c6a3 lrc: 3/0,1 mode: --/CW res: [0x240000405:0x7:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66090 initiator: MDT0 00000020:00100000:1.0:1769074838.307124:0:66090:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074838.307137:0:66090:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342cea680 x1855007617005696/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/67888 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074838.307146:0:66090:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0344ce0e00/0xb7d6bf619879c6a3 lrc: 3/0,1 mode: --/CW res: [0x240000405:0x7:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66090 initiator: MDT0 00000100:00100000:1.0:1769074838.307166:0:66090:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66090:1855007617005696:192.168.204.160@tcp:101:dd.0 00000100:00100000:1.0:1769074838.307443:0:66090:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0342f620c0 going to sleep for 50 seconds 00000080:00200008:0.0:1769074838.510025:0:66093:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074838.510036:0:66093:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074838.510062:0:66093:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074838.510080:0:66093:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.510096:0:66093:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.510104:0:66093:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074838.510112:0:66093:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074838.510115:0:66093:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074838.510123:0:66093:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb436414f3cb8 00010000:00010000:0.0:1769074838.510126:0:66093:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.510135:0:66093:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.510389:0:66093:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074838.510399:0:66093:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074838.510406:0:66093:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.510415:0:66093:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.510421:0:66093:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074838.510425:0:66093:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074838.510428:0:66093:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074838.510433:0:66093:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb436414f3bb0 00010000:00010000:0.0:1769074838.510436:0:66093:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074838.510443:0:66093:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074838.510452:0:66093:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074838.510460:0:66093:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:0.0:1769074838.510477:0:66093:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7fff50c5e63c 00000080:00200008:0.0:1769074838.510490:0:66093:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200008:1.0:1769074838.593760:0:66100:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074838.593770:0:66100:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074838.593792:0:66100:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074838.593810:0:66100:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074838.593825:0:66100:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074838.593832:0:66100:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074838.593839:0:66100:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074838.593842:0:66100:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:1.0:1769074838.593849:0:66100:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb4364176bcb8 00010000:00010000:1.0:1769074838.593852:0:66100:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074838.593860:0:66100:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074838.593944:0:66100:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074838.593952:0:66100:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074838.593958:0:66100:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074838.593966:0:66100:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074838.593973:0:66100:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074838.593976:0:66100:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074838.593979:0:66100:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:1.0:1769074838.593982:0:66100:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb4364176bbb0 00010000:00010000:1.0:1769074838.593985:0:66100:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074838.594006:0:66100:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074838.594014:0:66100:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074838.594020:0:66100:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074838.594035:0:66100:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7fff11bfb07c 00000080:00200008:1.0:1769074838.594046:0:66100:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000100:00100000:1.0:1769074838.611600:0:66102:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66102:1855007617008384:192.168.204.160@tcp:34:lfs.0 00000080:00200008:1.0:1769074838.611765:0:66102:0:(file.c:424:ll_file_release()) START file d204a.sanity-flr:[0x200000405:0x205:0x0](ffff9b0355d47448), flags 304000 00010000:00010000:1.0:1769074838.611797:0:66102:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c6cd lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x205:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55233 expref: -99 pid: 66097 timeout: 0 lvb_type: 0 00000100:00100000:1.0:1769074838.664759:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074838.664897:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625226880 00000100:00100000:0.0:1769074838.664908:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625226880 00000100:00100000:0.0:1769074838.665005:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8930 00000100:00100000:0.0:1769074838.665014:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:28519:x1855007625226880:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074838.665032:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1c00/0xb7d6bf619879c6e2 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd5529c expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074838.665106:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074838.665113:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:28519:x1855007625226880:12345-192.168.204.160@tcp:104: Request processed in 101us (366us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074838.665121:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8930 00010000:00010000:0.0:1769074838.671754:0:66106:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1400/0xb7d6bf619879c6db lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55295 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074838.671781:0:66106:0:(vvp_object.c:114:vvp_conf_set()) [0x240000404:0x4b:0x0]: losing layout lock 00010000:00010000:0.0:1769074838.671802:0:66106:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c6aa lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4b:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd551c3 expref: -99 pid: 66090 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074838.671830:0:66106:0:(vvp_object.c:114:vvp_conf_set()) [0x240000404:0x4b:0x0]: losing layout lock 00010000:00010000:0.0:1769074838.671862:0:66106:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1400/0xb7d6bf619879c6db lrc: 1/0,0 mode: --/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55295 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074838.671870:0:66106:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c6aa lrc: 1/0,0 mode: --/CR res: [0x240000404:0x4b:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd551c3 expref: -99 pid: 66090 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074838.671878:0:66106:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074838.671881:0:66106:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1400/0xb7d6bf619879c6db lrc: 0/0,0 mode: --/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55295 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074838.671895:0:66106:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c6aa lrc: 0/0,0 mode: --/CR res: [0x240000404:0x4b:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd551c3 expref: -99 pid: 66090 timeout: 0 lvb_type: 3 00000020:00100000:0.0:1769074838.671910:0:66106:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074838.671919:0:66106:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a38380 x1855007617009792/t0(0) o36->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 472/440 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074838.671943:0:66106:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66106:1855007617009792:192.168.204.160@tcp:36:lfs.0 00000100:00100000:0.0:1769074838.672032:0:66106:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ad80 going to sleep for 50 seconds 00000100:00100000:0.0:1769074838.703210:0:66106:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9b0342ad2800: committing for last_committed 4294968360 gen 1 00000100:00100000:0.0:1769074838.703231:0:66106:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff9b0352a19f80 x1855007617008896/t4294968358(4294968358) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074893 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1769074838.703414:0:66106:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff9b0352a19500 x1855007617009152/t4294968359(4294968359) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074893 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1769074838.703425:0:66106:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66106:1855007617009792:192.168.204.160@tcp:36:lfs.0 00000080:00200080:1.0:1769074838.813951:0:66106:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4b:0x0](ffff9b03606fa1c8) cmd=400866b1 arg=1 00000080:00200008:1.0:1769074838.813972:0:66106:0:(file.c:6252:ll_getattr_dentry()) START file f204a.sanity-flr:[0x240000404:0x4b:0x0](ffff9b03606fa1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074838.813980:0:66106:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x240000404:0x4b:0x0](ffff9b03606fa1c8),name=f204a.sanity-flr 00000002:00010000:1.0:1769074838.814023:0:66106:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4b:0x0]) in obj [0x240000404:0x4b:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074838.814043:0:66106:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c6fe lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd552e9 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074838.814058:0:66106:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c6fe lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd552e9 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074838.814067:0:66106:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000404:0x4b:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074838.814076:0:66106:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4b:0x0](ffff9b03606fa1c8) 00000080:00010000:1.0:1769074838.814079:0:66106:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4b:0x0](ffff9b03606fa1c8) for lock 0xb7d6bf619879c6fe 00000080:00010000:1.0:1769074838.814088:0:66106:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c6fe from it ffffb43641c17cd8 00010000:00010000:1.0:1769074838.814091:0:66106:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c6fe lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd552e9 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074838.814099:0:66106:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c6fe lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd552e9 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074838.814127:0:66106:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4b:0x0] ignore/verify layout 0/1, layout version 3 restore needed 0 00020000:00200000:1.0:1769074838.814142:0:66106:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4b:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074838.814145:0:66106:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1769074838.814149:0:66106:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240000404:0x4b:0x0] 00000008:00010000:1.0:1769074838.814169:0:66106:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074838.814172:0:66106:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360af6730, osc lock ffff9b0345a08000, flags 0x400000400001002 00010000:00010000:1.0:1769074838.816795:0:66106:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.816819:0:66106:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074838.816839:0:66106:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074838.816843:0:66106:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.816851:0:66106:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074838.816868:0:66106:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074838.816882:0:66106:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4b:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1769074838.816887:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03531286d8 (ffff9b0342b7de18/0) at cl_lock_release():245 00000020:00010000:1.0:1769074838.816891:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03531286d8 (ffff9b0342b7de18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074838.816896:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360af6730 (ffff9b0342b7d3f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074838.816901:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03531286d8 (ffff9b0342b7de18/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074838.816903:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360af6730 (ffff9b0342b7de18/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074838.816918:0:66106:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4b:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074838.816945:0:66106:0:(file.c:5185:ll_file_seek()) START file f204a.sanity-flr:[0x240000404:0x4b:0x0], offset: 0, type: SEEK_DATA 00000080:00200000:1.0:1769074838.816952:0:66106:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1769074838.816962:0:66106:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4b:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074838.816964:0:66106:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1769074838.816970:0:66106:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1769074838.816979:0:66106:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1769074838.817004:0:66106:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4b:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:1.0:1769074838.817013:0:66106:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074838.817018:0:66106:0:(file.c:1759:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 1 00000080:00200000:1.0:1769074838.817020:0:66106:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074838.817025:0:66106:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1769074838.817029:0:66106:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1769074838.817031:0:66106:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:1.0:1769074838.817046:0:66106:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1769074838.817049:0:66106:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360af6730, osc lock ffff9b0345a08000, flags 0x0 00010000:00010000:1.0:1769074838.817061:0:66106:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.817072:0:66106:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.817081:0:66106:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.817089:0:66106:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.817097:0:66106:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074838.817107:0:66106:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4b:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:0.0:1769074838.818366:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617010304:192.168.204.160@tcp:23:lfs.0 00000100:00100000:0.0:1769074838.835316:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617010304:192.168.204.160@tcp:23:lfs.0 00000020:00010000:1.0:1769074838.835559:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03590b0f38 (ffff9b0342b7de18/0) at cl_lock_release():245 00000020:00010000:1.0:1769074838.835564:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03590b0f38 (ffff9b0342b7de18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074838.835571:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360af6730 (ffff9b0342b7d3f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074838.835581:0:66106:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.835595:0:66106:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074838.835616:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03590b0f38 (ffff9b0342b7de18/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074838.835619:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360af6730 (ffff9b0342b7de18/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074838.835635:0:66106:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074838.835648:0:66106:0:(file.c:5216:ll_file_seek()) COMPLETED file f204a.sanity-flr:[0x240000404:0x4b:0x0], offset: 0, type: SEEK_DATA, rc = 0 00000080:00200008:1.0:1769074838.835659:0:66106:0:(file.c:5185:ll_file_seek()) START file f204a.sanity-flr:[0x240000404:0x4b:0x0], offset: 0, type: SEEK_HOLE 00000080:00200000:1.0:1769074838.835667:0:66106:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1769074838.835676:0:66106:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4b:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074838.835679:0:66106:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1769074838.835684:0:66106:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1769074838.835705:0:66106:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1769074838.835729:0:66106:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4b:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:1.0:1769074838.835736:0:66106:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074838.835741:0:66106:0:(file.c:1759:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 1 00000080:00200000:1.0:1769074838.835744:0:66106:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074838.835749:0:66106:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1769074838.835752:0:66106:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1769074838.835754:0:66106:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:1.0:1769074838.835766:0:66106:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1769074838.835770:0:66106:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360af6730, osc lock ffff9b0345a08000, flags 0x0 00010000:00010000:1.0:1769074838.835790:0:66106:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.835803:0:66106:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.835815:0:66106:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.835828:0:66106:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.835835:0:66106:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074838.835847:0:66106:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4b:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:0.0:1769074838.836036:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617010432:192.168.204.160@tcp:23:lfs.0 00000100:00100000:0.0:1769074838.854072:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617010432:192.168.204.160@tcp:23:lfs.0 00000020:00010000:1.0:1769074838.854356:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03590b0f38 (ffff9b0342b7de18/0) at cl_lock_release():245 00000020:00010000:1.0:1769074838.854361:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03590b0f38 (ffff9b0342b7de18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074838.854368:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360af6730 (ffff9b0342b7d3f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1769074838.854379:0:66106:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.854392:0:66106:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00000020:00010000:1.0:1769074838.854409:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03590b0f38 (ffff9b0342b7de18/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074838.854411:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360af6730 (ffff9b0342b7de18/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074838.854429:0:66106:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074838.854441:0:66106:0:(file.c:5216:ll_file_seek()) COMPLETED file f204a.sanity-flr:[0x240000404:0x4b:0x0], offset: 0, type: SEEK_HOLE, rc = 2097152 00000080:00200080:1.0:1769074838.854456:0:66106:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4b:0x0](ffff9b03606fa1c8) cmd=400866b1 arg=0 00000080:00200080:1.0:1769074838.854467:0:66106:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4b:0x0](ffff9b03606fa1c8) cmd=400866b1 arg=1 00000080:00200008:1.0:1769074838.854473:0:66106:0:(file.c:6252:ll_getattr_dentry()) START file f204a.sanity-flr:[0x240000404:0x4b:0x0](ffff9b03606fa1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074838.854480:0:66106:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x240000404:0x4b:0x0](ffff9b03606fa1c8),name=f204a.sanity-flr 00000002:00010000:1.0:1769074838.854503:0:66106:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4b:0x0]) in obj [0x240000404:0x4b:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074838.854521:0:66106:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c6fe lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd552e9 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074838.854533:0:66106:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c6fe lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd552e9 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074838.854541:0:66106:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000404:0x4b:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074838.854547:0:66106:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4b:0x0](ffff9b03606fa1c8) 00000080:00010000:1.0:1769074838.854551:0:66106:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4b:0x0](ffff9b03606fa1c8) for lock 0xb7d6bf619879c6fe 00000080:00010000:1.0:1769074838.854557:0:66106:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c6fe from it ffffb43641c17cd8 00010000:00010000:1.0:1769074838.854561:0:66106:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c6fe lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd552e9 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074838.854568:0:66106:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c6fe lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4b:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd552e9 expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074838.854585:0:66106:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4b:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:1.0:1769074838.854595:0:66106:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4b:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074838.854598:0:66106:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1769074838.854602:0:66106:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240000404:0x4b:0x0] 00000008:00010000:1.0:1769074838.854619:0:66106:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074838.854623:0:66106:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360af6730, osc lock ffff9b0345a08000, flags 0x400000400001002 00010000:00010000:1.0:1769074838.854637:0:66106:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.854646:0:66106:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074838.854656:0:66106:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074838.854660:0:66106:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074838.854668:0:66106:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074838.854676:0:66106:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074838.854786:0:66106:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4b:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1769074838.854792:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03531286d8 (ffff9b0342b7de18/0) at cl_lock_release():245 00000020:00010000:1.0:1769074838.854794:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03531286d8 (ffff9b0342b7de18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074838.854797:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360af6730 (ffff9b0342b7d3f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074838.854801:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03531286d8 (ffff9b0342b7de18/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074838.854813:0:66106:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360af6730 (ffff9b0342b7de18/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074838.854820:0:66106:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4b:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074838.854842:0:66106:0:(file.c:2474:do_file_read_iter()) START file f204a.sanity-flr:[0x240000404:0x4b:0x0], ppos: 0, count: 2097152 00000080:00200000:1.0:1769074838.854846:0:66106:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1769074838.854851:0:66106:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4b:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074838.854854:0:66106:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:1.0:1769074838.854858:0:66106:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074838.854866:0:66106:0:(file.c:1994:ll_file_io_generic()) f204a.sanity-flr: read ppos: 0, bytes: 2097152 00000080:00200000:1.0:1769074838.854874:0:66106:0:(file.c:1759:ll_io_set_mirror()) f204a.sanity-flr: desiginated mirror: 1 00000020:00200000:1.0:1769074838.854881:0:66106:0:(cl_io.c:214:cl_io_rw_init()) header@00000000a4bf14b4[0x4, 2, [0x240000404:0x4b:0x0] hash] 00000020:00200000:1.0:1769074838.854883:0:66106:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 2097152) 2048 0 00000080:00200000:1.0:1769074838.854885:0:66106:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4b:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:1.0:1769074838.854893:0:66106:0:(file.c:2072:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:1.0:1769074838.854900:0:66106:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2097152) 2097152, 2097152 00020000:00200000:1.0:1769074838.854904:0:66106:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074838.854915:0:66106:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 511] 00000008:00010000:1.0:1769074838.854924:0:66106:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:1.0:1769074838.854928:0:66106:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360af6730, osc lock ffff9b0345a08000, flags 0x40000 00000080:00200000:1.0:1769074838.854933:0:66106:0:(vvp_io.c:827:vvp_io_read_start()) f204a.sanity-flr: read [0, 2097152) 00000080:00200000:1.0:1769074838.854940:0:66106:0:(vvp_io.c:886:vvp_io_read_start()) count:2097152 iocb pos:0 00000080:00200000:1.0:1769074838.854947:0:66106:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x240000404:0x4b:0x0](ffff9b03606fa1c8), size=2097152 (max 603979776), offset=0=0x0, pages 512 (max 147456), lockless, parallel 00010000:00010000:0.0:1769074838.931541:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074839.124598:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617011456:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074839.124693:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074839.124699:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617011328:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074839.132476:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617011328:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074839.132501:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617011456:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074839.151362:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074839.151447:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625227392 00000100:00100000:0.0:1769074839.151460:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625227392 00000100:00100000:0.0:1769074839.151524:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8931 00000100:00100000:0.0:1769074839.151536:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ceb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:36516:x1855007625227392:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074839.151554:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c705 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x35:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd552f0 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074839.151633:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074839.151640:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ceb800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:36516:x1855007625227392:12345-192.168.204.160@tcp:104: Request processed in 107us (290us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074839.151647:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8931 00000100:00100000:0.0:1769074839.155963:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074839.156088:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625227520 00000100:00100000:1.0:1769074839.156095:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625227520 00000100:00100000:1.0:1769074839.156148:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8932 00000100:00100000:1.0:1769074839.156157:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a38a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6500:x1855007625227520:12345-192.168.204.160@tcp:105: 00010000:00010000:1.0:1769074839.156168:0:39330:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 5/0,1 mode: --/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074839.156177:0:39330:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 5/0,1 mode: --/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074839.156186:0:39330:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 5/0,1 mode: --/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000020000 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074839.156195:0:39330:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10080000020000 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074839.156205:0:39330:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10080000020000 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074839.156214:0:39330:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 6/0,1 mode: PW/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x80000020000 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074839.156262:0:39330:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b0342cd1600) 00000100:00100000:1.0:1769074839.156314:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a38a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6500:x1855007625227520:12345-192.168.204.160@tcp:105: Request processed in 158us (517us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074839.156322:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8932 00000100:00100000:1.0:1769074839.167220:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074839.169182:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625227648 00000100:00100000:0.0:1769074839.169199:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625227648 00000100:00100000:0.0:1769074839.169295:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8933 00000100:00100000:0.0:1769074839.169305:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:15947:x1855007625227648:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074839.169322:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c6f7 lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4b:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd552db expref: -99 pid: 66106 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074839.169412:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074839.169419:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:15947:x1855007625227648:12345-192.168.204.160@tcp:104: Request processed in 116us (2210us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074839.169427:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8933 00000080:00200008:1.0:1769074839.250890:0:66108:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074839.250901:0:66108:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074839.250925:0:66108:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074839.250943:0:66108:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074839.250960:0:66108:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074839.250968:0:66108:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074839.250976:0:66108:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074839.259118:0:66108:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:1.0:1769074839.259146:0:66108:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb436420d3cb8 00010000:00010000:1.0:1769074839.259153:0:66108:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074839.259168:0:66108:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074839.259349:0:66108:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074839.259382:0:66108:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074839.259397:0:66108:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074839.259411:0:66108:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074839.259418:0:66108:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074839.259424:0:66108:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074839.259427:0:66108:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:1.0:1769074839.259431:0:66108:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb436420d3bb0 00010000:00010000:1.0:1769074839.259434:0:66108:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074839.259441:0:66108:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074839.259450:0:66108:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074839.259458:0:66108:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074839.259637:0:66108:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7fff7fb3961c 00000080:00200008:1.0:1769074839.259649:0:66108:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:1.0:1769074839.321139:0:66117:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074839.321173:0:66117:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074839.321191:0:66117:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074839.321208:0:66117:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074839.321216:0:66117:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074839.321224:0:66117:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074839.321227:0:66117:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:1.0:1769074839.321257:0:66117:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb43641ca7bf8 00010000:00010000:1.0:1769074839.321260:0:66117:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074839.321269:0:66117:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074839.321283:0:66117:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074839.321291:0:66117:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=27 00000080:00200000:1.0:1769074839.321297:0:66117:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x205:0x0](ffff9b0355d47448), inode mode 41ed mask 1 00000080:00200000:1.0:1769074839.321307:0:66117:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204a.sanity-flr, dir=[0x200000405:0x205:0x0](ffff9b0355d47448), file ffff9b0342fbc900, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1769074839.321313:0:66117:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204a.sanity-flr, dir=[0x200000405:0x205:0x0](ffff9b0355d47448), intent=open 00000002:00010000:1.0:1769074839.321328:0:66117:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204a.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0x7:0x0], intent: open flags 0304001 00010000:00010000:1.0:1769074839.321385:0:66117:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 3/1,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66117 initiator: MDT0 00010000:00010000:1.0:1769074839.321392:0:66117:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 3/1,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66117 initiator: MDT0 00000020:00100000:1.0:1769074839.321404:0:66117:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074839.321416:0:66117:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ce9c00 x1855007617013760/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074839.321424:0:66117:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 3/1,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66117 initiator: MDT0 00000100:00100000:1.0:1769074839.321445:0:66117:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66117:1855007617013760:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074839.321622:0:66117:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0342f629c0 going to sleep for 50 seconds 00000100:00100000:1.0:1769074839.325933:0:66117:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66117:1855007617013760:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074839.325960:0:66117:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 4/1,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66117 initiator: MDT0 00010000:00010000:1.0:1769074839.325972:0:66117:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 4/1,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 66117 initiator: MDT0 00010000:00010000:1.0:1769074839.326052:0:66117:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 4/1,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 66117 initiator: MDT0 00010000:00010000:1.0:1769074839.326061:0:66117:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 3/0,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 66117 initiator: MDT0 00010000:00010000:1.0:1769074839.326069:0:66117:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 4/0,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 66117 initiator: MDT0 00010000:00010000:1.0:1769074839.326075:0:66117:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b034164ce00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:1.0:1769074839.326080:0:66117:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 4/0,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 66117 initiator: MDT0 00010000:00010000:1.0:1769074839.326087:0:66117:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 4/0,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 66117 initiator: MDT0 00010000:00010000:1.0:1769074839.326094:0:66117:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 5/0,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 66117 initiator: MDT0 00010000:00010000:1.0:1769074839.326101:0:66117:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074839.326107:0:66117:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 3/0,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 66117 initiator: MDT0 00010000:00010000:1.0:1769074839.326112:0:66117:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b034164ce00/0xb7d6bf619879c72f lrc: 0/0,0 mode: --/CR res: [0x240000405:0x7:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 66117 initiator: MDT0 00000002:00100000:1.0:1769074839.326138:0:66117:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0342ce9c00 x1855007617013760/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/608 e 0 to 0 dl 1769074894 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074839.326158:0:66117:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x240000405:0x7:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1769074839.330710:0:66117:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074839.330739:0:66117:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074839.330756:0:66117:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074839.330770:0:66117:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074839.330778:0:66117:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074839.330785:0:66117:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074839.330789:0:66117:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074839.330795:0:66117:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb43641ca7bf8 00010000:00010000:0.0:1769074839.330799:0:66117:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074839.330808:0:66117:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074839.330821:0:66117:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074839.330828:0:66117:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=267 00000080:00200000:0.0:1769074839.330835:0:66117:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x205:0x0](ffff9b0355d47448), inode mode 41ed mask 44 00000080:00200008:0.0:1769074839.330843:0:66117:0:(file.c:1001:ll_file_open()) START file d204a.sanity-flr:[0x200000405:0x205:0x0](ffff9b0355d47448), flags 304000 00000080:00200080:0.0:1769074839.330884:0:66117:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x205:0x0](ffff9b0355d47448) cmd=c1406916 arg=2656a10 00000100:00100000:0.0:1769074839.330948:0:66117:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66117:1855007617013888:192.168.204.160@tcp:34:lfs.0 00000100:00100000:0.0:1769074839.331083:0:66117:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ae40 going to sleep for 50 seconds 00000100:00100000:1.0:1769074839.339101:0:66117:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66117:1855007617013888:192.168.204.160@tcp:34:lfs.0 00000080:00200008:1.0:1769074839.339298:0:66117:0:(file.c:424:ll_file_release()) START file d204a.sanity-flr:[0x200000405:0x205:0x0](ffff9b0355d47448), flags 304000 00010000:00010000:1.0:1769074839.339330:0:66117:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0000/0xb7d6bf619879c6cd lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x205:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55233 expref: -99 pid: 66097 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074839.368964:0:66122:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074839.368975:0:66122:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074839.369011:0:66122:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074839.369027:0:66122:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074839.369042:0:66122:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074839.369050:0:66122:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074839.369056:0:66122:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074839.369060:0:66122:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:1.0:1769074839.369067:0:66122:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb43641c9fcb8 00010000:00010000:1.0:1769074839.369071:0:66122:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074839.369078:0:66122:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074839.369177:0:66122:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074839.369186:0:66122:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074839.369194:0:66122:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074839.369208:0:66122:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074839.369213:0:66122:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074839.369217:0:66122:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074839.369220:0:66122:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:1.0:1769074839.369224:0:66122:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb43641c9fbb0 00010000:00010000:1.0:1769074839.369227:0:66122:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074839.369256:0:66122:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074839.369265:0:66122:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074839.369272:0:66122:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074839.369288:0:66122:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffd3fbf89dc 00000080:00200008:1.0:1769074839.369302:0:66122:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200008:0.0:1769074839.455910:0:66129:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074839.455923:0:66129:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074839.455948:0:66129:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074839.455965:0:66129:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074839.455991:0:66129:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074839.455999:0:66129:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074839.456007:0:66129:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074839.456010:0:66129:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074839.456017:0:66129:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb43641c97cb8 00010000:00010000:0.0:1769074839.456020:0:66129:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074839.456027:0:66129:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074839.456180:0:66129:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074839.456189:0:66129:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074839.456196:0:66129:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074839.456204:0:66129:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074839.456211:0:66129:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074839.456215:0:66129:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074839.456217:0:66129:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074839.456221:0:66129:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb43641c97bb0 00010000:00010000:0.0:1769074839.456224:0:66129:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074839.456230:0:66129:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074839.457074:0:66129:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074839.457084:0:66129:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:0.0:1769074839.457108:0:66129:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffdf4782f8c 00000080:00200008:0.0:1769074839.457123:0:66129:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00010000:00010000:0.0:1769074844.055643:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074844.244510:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617015296:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074844.244657:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617015552:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074844.244696:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1769074844.244703:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617015168:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074844.244731:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617015424:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074844.244752:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617015680:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074844.279932:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617015168:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074844.279957:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617015424:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074844.279971:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617015680:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074844.279982:0:2406:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944097 gen 1 00000100:00100000:0.0:1769074844.280002:0:2406:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff9b0352a19180 x1855007617007872/t8589944097(8589944097) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074893 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1769074844.280012:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617015296:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074844.280025:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617015552:192.168.204.160@tcp:400:kworker.0 00000080:00200000:0.0:1769074844.597107:0:66558:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074844.597135:0:66558:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074844.597153:0:66558:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074844.597168:0:66558:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074844.597177:0:66558:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074844.597183:0:66558:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074844.597187:0:66558:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074844.597215:0:66558:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb4364145bb58 00010000:00010000:0.0:1769074844.597219:0:66558:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074844.597227:0:66558:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074844.597366:0:66558:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074844.597374:0:66558:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204a.sanity-flr, flags=24 00000100:00100000:1.0:1769074844.674148:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074844.674189:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074844.674286:0:39330:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074844.674311:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625230848 00000100:00100000:0.0:1769074844.674321:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625230848 00000100:00100000:0.0:1769074844.674371:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625230976 00000100:00100000:0.0:1769074844.674374:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625230976 00000100:00100000:0.0:1769074844.674391:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8934 00000100:00100000:0.0:1769074844.674400:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cea300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28955:x1855007625230848:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074844.674418:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c70c lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55313 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674496:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074844.674503:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cea300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28955:x1855007625230848:12345-192.168.204.160@tcp:104: Request processed in 104us (366us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074844.674510:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8934 00000100:00100000:0.0:1769074844.674527:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8935 00000100:00100000:0.0:1769074844.674533:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0348c2f850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8398:x1855007625230976:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074844.674542:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674573:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074844.674578:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0348c2f850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8398:x1855007625230976:12345-192.168.204.160@tcp:104: Request processed in 46us (390us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074844.674585:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8935 00010000:00010000:0.0:1769074844.674696:0:60220:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1769074844.674702:0:60220:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c70c lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55313 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674723:0:60220:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860a00 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:0.0:1769074844.674729:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c70c lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x428400010000 nid: local remote: 0x7fe2323adfd55313 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674770:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c70c lrc: 2/0,0 mode: --/PW res: [0x2c0000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55313 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674778:0:60220:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074844.674799:0:60220:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c70c lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55313 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674806:0:60220:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c70c lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55313 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674827:0:60220:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074844.674831:0:60220:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674837:0:60220:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0342cd1600 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:0.0:1769074844.674841:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x428400030000 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674859:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 2/0,0 mode: --/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674864:0:60220:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074844.674870:0:60220:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 1/0,0 mode: --/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674876:0:60220:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c71a lrc: 0/0,0 mode: --/PW res: [0x280000400:0x35:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55328 expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074844.674898:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074844.674900:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074844.674902:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074844.674908:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074844.674922:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617016192:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074844.674982:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1769074844.674989:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617016320:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074844.693712:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617016320:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074844.696477:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617016192:192.168.204.160@tcp:103:ldlm_bl.0 00000001:02000400:1.0:1769074846.345970:0:66700:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204b: FLR write/stale/resync tests with self-extending primary ========================================================== 04:40:44 (1769074844) 00000080:00200000:0.0:1769074846.678393:0:66723:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074846.678423:0:66723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074846.678440:0:66723:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.678456:0:66723:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.678465:0:66723:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074846.678472:0:66723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074846.678476:0:66723:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074846.678482:0:66723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb4364176bb58 00010000:00010000:0.0:1769074846.678486:0:66723:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.678494:0:66723:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.678511:0:66723:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074846.678526:0:66723:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d204b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), flags=13 00000080:00200000:0.0:1769074846.678531:0:66723:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d204b.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=getattr 00000002:00010000:0.0:1769074846.678544:0:66723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074846.678589:0:66723:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.678597:0:66723:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.678607:0:66723:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66723 initiator: MDT0 00000100:00100000:0.0:1769074846.678634:0:66723:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66723:1855007617016576:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074846.678738:0:66723:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ae40 going to sleep for 50 seconds 00000100:00100000:0.0:1769074846.684796:0:66723:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66723:1855007617016576:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074846.684819:0:66723:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.684833:0:66723:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.684839:0:66723:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.684845:0:66723:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.684851:0:66723:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.684867:0:66723:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0349292e00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074846.684872:0:66723:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.684879:0:66723:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.684886:0:66723:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.684893:0:66723:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074846.684900:0:66723:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66723 initiator: MDT0 00010000:00010000:0.0:1769074846.684905:0:66723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349292e00/0xb7d6bf619879c752 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66723 initiator: MDT0 00000002:00100000:0.0:1769074846.684923:0:66723:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9b0365a3b800 x1855007617016576/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074901 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074846.684937:0:66723:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1769074846.684966:0:66723:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.684978:0:66723:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00010000:0.0:1769074846.684986:0:66723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb4364176b940 00010000:00010000:0.0:1769074846.684990:0:66723:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.685008:0:66723:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.685083:0:66723:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074846.685097:0:66723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074846.685104:0:66723:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.685112:0:66723:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.685118:0:66723:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074846.685122:0:66723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074846.685125:0:66723:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074846.685130:0:66723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb4364176bbe8 00010000:00010000:0.0:1769074846.685133:0:66723:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.685139:0:66723:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.685157:0:66723:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074846.685174:0:66723:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=16392 00000080:00200008:0.0:1769074846.685202:0:66723:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074846.685206:0:66723:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074846.685213:0:66723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074846.685219:0:66723:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.685227:0:66723:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.685262:0:66723:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074846.685266:0:66723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074846.685269:0:66723:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074846.685272:0:66723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb4364176bcb8 00010000:00010000:0.0:1769074846.685275:0:66723:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.685282:0:66723:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200008:0.0:1769074846.685847:0:66723:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074846.685853:0:66723:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074846.685873:0:66723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074846.685880:0:66723:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.685890:0:66723:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.685897:0:66723:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074846.685903:0:66723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074846.685905:0:66723:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074846.685910:0:66723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb4364176bcb8 00010000:00010000:0.0:1769074846.685913:0:66723:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.685919:0:66723:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.686046:0:66723:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074846.686056:0:66723:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074846.686063:0:66723:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.686071:0:66723:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.686077:0:66723:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074846.686081:0:66723:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074846.686084:0:66723:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c66b 00000080:00010000:0.0:1769074846.686088:0:66723:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c66b from it ffffb4364176bbb0 00010000:00010000:0.0:1769074846.686091:0:66723:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.686098:0:66723:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.686106:0:66723:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074846.686113:0:66723:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:0.0:1769074846.686131:0:66723:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=c03066f0 arg=7ffef61561d0 00000080:00200000:0.0:1769074846.686143:0:66723:0:(dir.c:483:ll_dir_setdirstripe()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) name=d204b.sanity-flr stripe_offset=0 stripe_count=2, hash_type=2 00010000:00010000:0.0:1769074846.686167:0:66723:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.686222:0:66723:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.686229:0:66723:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074846.686253:0:66723:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0349293200/0xb7d6bf619879c66b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd550ff expref: -99 pid: 66089 timeout: 0 lvb_type: 0 00000020:00100000:0.0:1769074846.686280:0:66723:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074846.686292:0:66723:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3b800 x1855007617016832/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 568/2080 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074846.686314:0:66723:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66723:1855007617016832:192.168.204.160@tcp:36:lfs.0 00000100:00100000:0.0:1769074846.686408:0:66723:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ae40 going to sleep for 50 seconds 00000100:00100000:0.0:1769074846.702621:0:66723:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944098 gen 1 00000100:00100000:0.0:1769074846.702641:0:66723:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66723:1855007617016832:192.168.204.160@tcp:36:lfs.0 00000080:00200000:0.0:1769074846.702709:0:66723:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8) 00000080:00200000:0.0:1769074846.702716:0:66723:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x206:0x0], updating i_size 4096 00000080:00200008:0.0:1769074846.702760:0:66723:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:1.0:1769074846.750090:0:66726:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074846.750126:0:66726:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074846.750288:0:66726:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66726 initiator: MDT0 00010000:00010000:1.0:1769074846.750301:0:66726:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66726 initiator: MDT0 00010000:00010000:1.0:1769074846.750311:0:66726:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66726 initiator: MDT0 00000100:00100000:1.0:1769074846.750337:0:66726:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66726:1855007617016960:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074846.750900:0:66726:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210600 going to sleep for 50 seconds 00000100:00100000:1.0:1769074846.780832:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074846.780958:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625232896 00000100:00100000:0.0:1769074846.780970:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625232896 00000100:00100000:0.0:1769074846.781479:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8936 00000100:00100000:0.0:1769074846.781491:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:28955:x1855007625232896:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074846.781508:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861a00/0xb7d6bf619879c713 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd5531a expref: -99 pid: 66106 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074846.781585:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074846.781593:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:28955:x1855007625232896:12345-192.168.204.160@tcp:104: Request processed in 106us (772us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074846.781602:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8936 00000100:00100000:1.0:1769074846.805050:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074846.805186:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625233024 00000100:00100000:0.0:1769074846.805196:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625233024 00000100:00100000:0.0:1769074846.805420:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8937 00000100:00100000:0.0:1769074846.805431:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:10283:x1855007625233024:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074846.805447:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c783 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd55519 expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074846.805517:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074846.805523:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:10283:x1855007625233024:12345-192.168.204.160@tcp:104: Request processed in 96us (486us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074846.805530:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8937 00000080:00200000:1.0:1769074846.841816:0:66727:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074846.841957:0:66727:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074846.841978:0:66727:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074846.842087:0:66727:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074846.842103:0:66727:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074846.842111:0:66727:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074846.842115:0:66727:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074846.842122:0:66727:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43640dabbf8 00010000:00010000:1.0:1769074846.842126:0:66727:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074846.842134:0:66727:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074846.842149:0:66727:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074846.842759:0:66727:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=25 00000080:00200000:1.0:1769074846.842767:0:66727:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074846.842780:0:66727:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 3 00000080:00200000:1.0:1769074846.842785:0:66727:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), file ffff9b0342b71680, open_flags 8041, mode 81b6 opened 0 00000080:00200000:1.0:1769074846.842794:0:66727:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), intent=open|creat 00000002:00010000:1.0:1769074846.842816:0:66727:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204b.sanity-flr,[0x200000405:0x208:0x0]) in obj [0x200000bd0:0x8:0x0], intent: open|creat flags 0100102 00010000:00010000:1.0:1769074846.843510:0:66727:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d400/0xb7d6bf619879c791 lrc: 3/0,1 mode: --/CW res: [0x200000bd0:0x8:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66727 initiator: MDT0 00010000:00010000:1.0:1769074846.843521:0:66727:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d400/0xb7d6bf619879c791 lrc: 3/0,1 mode: --/CW res: [0x200000bd0:0x8:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66727 initiator: MDT0 00000020:00100000:1.0:1769074846.843533:0:66727:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074846.843545:0:66727:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ceb480 x1855007617019136/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/67824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074846.843557:0:66727:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d400/0xb7d6bf619879c791 lrc: 3/0,1 mode: --/CW res: [0x200000bd0:0x8:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66727 initiator: MDT0 00000100:00100000:1.0:1769074846.843583:0:66727:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66727:1855007617019136:192.168.204.160@tcp:101:dd.0 00000100:00100000:1.0:1769074846.843982:0:66727:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210480 going to sleep for 50 seconds 00000100:00100000:1.0:1769074846.850755:0:66727:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66727:1855007617019136:192.168.204.160@tcp:101:dd.0 00010000:00010000:1.0:1769074846.850785:0:66727:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d400/0xb7d6bf619879c791 lrc: 4/0,1 mode: --/CW res: [0x200000bd0:0x8:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55551 expref: -99 pid: 66727 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074846.882441:0:66727:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [256, 511] 00000008:00010000:0.0:1769074846.882456:0:66727:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1769074846.882462:0:66727:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568dff30, osc lock ffff9b0345a6d430, flags 0x0 00000080:00200000:0.0:1769074846.882473:0:66727:0:(vvp_io.c:1312:vvp_io_write_start()) f204b.sanity-flr: write [1048576, 2097152) append 0 append lockpos 0 00000080:00200000:0.0:1769074846.882481:0:66727:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988), size=1048576 (max 603979776), offset=1048576=0x100000, pages 256 (max 147456), lockless, parallel, hybrid 00000020:00200000:0.0:1769074846.882820:0:66727:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:0.0:1769074846.882823:0:66727:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 1048576 count 1048576 file_offset 2097152 00000020:00010000:0.0:1769074846.882828:0:66727:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03590b06c8 (ffff9b0342b7dd88/0) at cl_lock_release():245 00000020:00010000:0.0:1769074846.882830:0:66727:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03590b06c8 (ffff9b0342b7dd88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074846.882833:0:66727:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568dff30 (ffff9b0342b7d5a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074846.882836:0:66727:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03590b06c8 (ffff9b0342b7dd88/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074846.882838:0:66727:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568dff30 (ffff9b0342b7dd88/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074846.882844:0:66727:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:0.0:1769074846.882848:0:66727:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000080:00200000:0.0:1769074846.944003:0:66727:0:(file.c:2109:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:0.0:1769074846.944032:0:66727:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 0/0, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074846.944037:0:66727:0:(file.c:2142:ll_file_io_generic()) f204b.sanity-flr: 2 io complete with rc: 0, result: 2097152, restart: 0 00000080:00200000:0.0:1769074846.944055:0:66727:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x207:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:0.0:1769074846.944061:0:66727:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074846.944072:0:66727:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 2097152 00000080:00200008:0.0:1769074846.944108:0:66727:0:(file.c:424:ll_file_release()) START file f204b.sanity-flr:[0x200000405:0x207:0x0](ffff9b0355d41988), flags 100001 00000002:00100000:0.0:1769074846.944178:0:66727:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0342ceb480 x1855007617019136/t8589944103(8589944103) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1088/1160 e 0 to 0 dl 1769074901 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074846.944191:0:66727:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074846.944196:0:66727:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3b800 x1855007617020032/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074846.944220:0:66727:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66727:1855007617020032:192.168.204.160@tcp:35:dd.0 00000100:00100000:0.0:1769074846.944439:0:66727:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1acc0 going to sleep for 11 seconds 00000100:00100000:0.0:1769074846.992840:0:66728:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5e840 going to sleep for 50 seconds 00000100:00100000:1.0:1769074847.000308:0:66728:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66728:1855007617020544:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074847.000335:0:66728:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c7ad lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66728 initiator: MDT0 00010000:00010000:1.0:1769074847.000348:0:66728:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c7ad lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66728 initiator: MDT0 00010000:00010000:1.0:1769074847.000355:0:66728:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c7ad lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66728 initiator: MDT0 00010000:00010000:1.0:1769074847.000363:0:66728:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c7ad lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 66728 initiator: MDT0 00010000:00010000:1.0:1769074847.000371:0:66728:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c7ad lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 66728 initiator: MDT0 00010000:00010000:1.0:1769074847.000377:0:66728:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365861c00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:1.0:1769074847.000382:0:66728:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c7ad lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66728 initiator: MDT0 00010000:00010000:1.0:1769074847.000387:0:66728:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c7ad lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 66728 initiator: MDT0 00010000:00010000:1.0:1769074847.000393:0:66728:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c7ad lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 66728 initiator: MDT0 00010000:00010000:1.0:1769074847.000401:0:66728:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074847.000407:0:66728:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c7ad lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66728 initiator: MDT0 00010000:00010000:1.0:1769074847.000412:0:66728:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c7ad lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 66728 initiator: MDT0 00000002:00100000:1.0:1769074847.000429:0:66728:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9b03482c4700 x1855007617020544/t8589944106(8589944106) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074901 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074847.000442:0:66728:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b03482c4700 x1855007617020544/t8589944106(8589944106) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074901 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074847.000460:0:66728:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074847.000494:0:66728:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x206:0x0], updating i_size 4096 00000002:00100000:1.0:1769074847.000510:0:66728:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b03482c4700 x1855007617020544/t8589944106(8589944106) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074901 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1769074847.000603:0:66728:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8) cmd=c0086915 arg=11313d0 00000100:00100000:1.0:1769074847.000660:0:66728:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceaa00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66728:1855007617020672:192.168.204.160@tcp:34:lfs.0 00000100:00100000:1.0:1769074847.000753:0:66728:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210480 going to sleep for 50 seconds 00000100:00100000:1.0:1769074847.003648:0:66728:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceaa00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66728:1855007617020672:192.168.204.160@tcp:34:lfs.0 00000080:00200008:1.0:1769074847.003970:0:66728:0:(file.c:424:ll_file_release()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00000002:00100000:1.0:1769074847.004035:0:66728:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b03482c4700 x1855007617020544/t8589944106(8589944106) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074901 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074847.004049:0:66728:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074847.004054:0:66728:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ceaa00 x1855007617020928/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074847.004082:0:66728:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceaa00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66728:1855007617020928:192.168.204.160@tcp:35:lfs.0 00000100:00100000:1.0:1769074847.004157:0:66728:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210480 going to sleep for 11 seconds 00000100:00100000:1.0:1769074847.006225:0:66728:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceaa00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66728:1855007617020928:192.168.204.160@tcp:35:lfs.0 00000002:00100000:1.0:1769074847.006466:0:66728:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b03482c4700 x1855007617020544/t8589944106(8589944106) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074901 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074847.045510:0:66732:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074847.045543:0:66732:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074847.045560:0:66732:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.045572:0:66732:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.045578:0:66732:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074847.045585:0:66732:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074847.045588:0:66732:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:0.0:1769074847.045594:0:66732:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641453bf8 00010000:00010000:0.0:1769074847.045596:0:66732:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.045602:0:66732:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.045613:0:66732:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074847.045620:0:66732:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:0.0:1769074847.045624:0:66732:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1769074847.045633:0:66732:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), file ffff9b03586ec900, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1769074847.045638:0:66732:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), intent=open 00000002:00010000:0.0:1769074847.045650:0:66732:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x8:0x0], intent: open flags 0304001 00010000:00010000:0.0:1769074847.045701:0:66732:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215800/0xb7d6bf619879c7b4 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66732 initiator: MDT0 00010000:00010000:0.0:1769074847.045709:0:66732:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215800/0xb7d6bf619879c7b4 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66732 initiator: MDT0 00000020:00100000:0.0:1769074847.045721:0:66732:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074847.045733:0:66732:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3b800 x1855007617021184/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074847.045742:0:66732:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215800/0xb7d6bf619879c7b4 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66732 initiator: MDT0 00000100:00100000:0.0:1769074847.045763:0:66732:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66732:1855007617021184:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074847.045965:0:66732:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ac00 going to sleep for 50 seconds 00000080:00200000:1.0:1769074847.055411:0:66732:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074847.055437:0:66732:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074847.055452:0:66732:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.055466:0:66732:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.055473:0:66732:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074847.055480:0:66732:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074847.055483:0:66732:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074847.055490:0:66732:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641453bf8 00010000:00010000:1.0:1769074847.055493:0:66732:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.055502:0:66732:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.055516:0:66732:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074847.055523:0:66732:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:1.0:1769074847.055530:0:66732:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 44 00000080:00200008:1.0:1769074847.055539:0:66732:0:(file.c:1001:ll_file_open()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00000002:00010000:1.0:1769074847.055552:0:66732:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x206:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:1.0:1769074847.055870:0:66732:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66732 initiator: MDT0 00010000:00010000:1.0:1769074847.055878:0:66732:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66732 initiator: MDT0 00000020:00100000:1.0:1769074847.055891:0:66732:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074847.055902:0:66732:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ceaa00 x1855007617021440/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074847.055911:0:66732:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66732 initiator: MDT0 00000100:00100000:1.0:1769074847.055933:0:66732:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceaa00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66732:1855007617021440:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074847.057462:0:66732:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210480 going to sleep for 50 seconds 00000100:00100000:1.0:1769074847.059984:0:66732:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceaa00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66732:1855007617021440:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074847.060086:0:66732:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 4/1,0 mode: --/CR res: [0x200000405:0x206:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.060097:0:66732:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 4/1,0 mode: --/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.060109:0:66732:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.060120:0:66732:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074847.060135:0:66732:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff9b0342ceaa00 x1855007617021440/t8589944108(8589944108) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074902 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074847.060148:0:66732:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0342ceaa00 x1855007617021440/t8589944108(8589944108) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074902 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074847.060163:0:66732:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.060174:0:66732:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 16 res 8589935621/518 (0 0) 00000080:00200000:1.0:1769074847.060180:0:66732:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074847.060454:0:66732:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x206:0x0], updating i_size 4096 00000080:00010000:1.0:1769074847.060462:0:66732:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x206:0x0](ffff9b03606fa1c8) for lock 0xb7d6bf619879c7bb 00000080:00010000:1.0:1769074847.060471:0:66732:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c7bb from it ffffb43641453bd0 00010000:00010000:1.0:1769074847.060475:0:66732:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.060485:0:66732:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074847.060510:0:66732:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0342ceaa00 x1855007617021440/t8589944108(8589944108) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074902 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:1.0:1769074847.060637:0:66732:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8) cmd=c1406916 arg=882a10 00000100:00100000:1.0:1769074847.060704:0:66732:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66732:1855007617021568:192.168.204.160@tcp:34:lfs.0 00000100:00100000:1.0:1769074847.060866:0:66732:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210480 going to sleep for 50 seconds 00000100:00100000:1.0:1769074847.068188:0:66732:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66732:1855007617021568:192.168.204.160@tcp:34:lfs.0 00000080:00200008:1.0:1769074847.068388:0:66732:0:(file.c:424:ll_file_release()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00010000:00010000:1.0:1769074847.068421:0:66732:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.124703:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1769074847.201537:0:66751:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074847.201546:0:66751:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074847.201567:0:66751:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074847.201582:0:66751:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.201595:0:66751:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.201601:0:66751:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074847.201607:0:66751:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074847.201610:0:66751:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:0.0:1769074847.201617:0:66751:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641453cb8 00010000:00010000:0.0:1769074847.201619:0:66751:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.201624:0:66751:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.201773:0:66751:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074847.201782:0:66751:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074847.201789:0:66751:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.201797:0:66751:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.201803:0:66751:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074847.201807:0:66751:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074847.201810:0:66751:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:0.0:1769074847.201814:0:66751:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641453bb0 00010000:00010000:0.0:1769074847.201817:0:66751:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.201824:0:66751:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.201833:0:66751:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074847.201840:0:66751:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:0.0:1769074847.201867:0:66751:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffe39d7d69c 00000080:00200008:0.0:1769074847.201966:0:66751:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:1.0:1769074847.248942:0:66757:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074847.248976:0:66757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074847.248996:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.249013:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.249022:0:66757:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074847.249029:0:66757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074847.249033:0:66757:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074847.249040:0:66757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641c77b58 00010000:00010000:1.0:1769074847.249044:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.249053:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.249070:0:66757:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074847.249078:0:66757:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=25 00000080:00200000:1.0:1769074847.249085:0:66757:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074847.249096:0:66757:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags=9 00000080:00200000:1.0:1769074847.249101:0:66757:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), intent=getattr 00000002:00010000:1.0:1769074847.249115:0:66757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x8:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074847.249167:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66757 initiator: MDT0 00010000:00010000:1.0:1769074847.249180:0:66757:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66757 initiator: MDT0 00010000:00010000:1.0:1769074847.249190:0:66757:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66757 initiator: MDT0 00000100:00100000:1.0:1769074847.249216:0:66757:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617022848:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074847.249348:0:66757:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210480 going to sleep for 50 seconds 00000100:00100000:1.0:1769074847.255081:0:66757:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617022848:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074847.255109:0:66757:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.255133:0:66757:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.255142:0:66757:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.255153:0:66757:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.255164:0:66757:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.255175:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.255183:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.255197:0:66757:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074847.255208:0:66757:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b0342ce9500 x1855007617022848/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/1136 e 0 to 0 dl 1769074902 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074847.255221:0:66757:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074847.255270:0:66757:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.255281:0:66757:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/519 (0 0) 00000080:00200000:1.0:1769074847.255287:0:66757:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x8:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074847.255335:0:66757:0:(namei.c:151:ll_iget()) got inode: [0x200000405:0x207:0x0](ffff9b0355d41988): rc = 0 00000080:00200000:1.0:1769074847.255350:0:66757:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x207:0x0]: layout version change: 6 -> 6 00000080:00010000:1.0:1769074847.255355:0:66757:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c7d7 00000080:00010000:1.0:1769074847.255365:0:66757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) 00000080:00010000:1.0:1769074847.255369:0:66757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c7d7 from it ffffb43641c77b20 00010000:00010000:1.0:1769074847.255372:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.255381:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200008:1.0:1769074847.255414:0:66757:0:(file.c:6252:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000405:0x207:0x0](ffff9b0355d41988), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074847.255420:0:66757:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988),name=f204b.sanity-flr 00000002:00010000:1.0:1769074847.255440:0:66757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x207:0x0]) in obj [0x200000bd0:0x8:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074847.255448:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.255458:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074847.255465:0:66757:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x8:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074847.255469:0:66757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) 00000080:00010000:1.0:1769074847.255472:0:66757:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c7d7 00000080:00010000:1.0:1769074847.255476:0:66757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c7d7 from it ffffb43641c77cb8 00010000:00010000:1.0:1769074847.255479:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.255486:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074847.255502:0:66757:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1769074847.255513:0:66757:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x207:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074847.255515:0:66757:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1769074847.255519:0:66757:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x207:0x0] 00000008:00010000:1.0:1769074847.255536:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074847.255540:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03492104b0, osc lock ffff9b0345a085f0, flags 0x400000400001002 00000008:00010000:1.0:1769074847.255546:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074847.255547:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03492104f0, osc lock ffff9b0345a08130, flags 0x400000400001002 00010000:00010000:1.0:1769074847.255587:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 3/1,0 mode: --/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.255596:0:66757:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 3/1,0 mode: --/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.255612:0:66757:0:(ldlm_request.c:1054:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff9b0342ce9500 x1855007617022976/t0(0) o101->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074847.255652:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.255659:0:66757:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 3/1,0 mode: --/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074847.255979:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617022976:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074847.256694:0:66757:0:(ldlm_request.c:1054:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff9b0342ceb480 x1855007617023104/t0(0) o101->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:1.0:1769074847.256718:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000100:00100000:0.0:1769074847.256998:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617023104:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074847.259045:0:2406:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 6/2,0 mode: --/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.259076:0:2406:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 6/2,0 mode: --/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.259089:0:2406:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.259101:0:2406:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074847.259117:0:2406:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074847.259132:0:2406:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074847.259136:0:2406:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.259143:0:2406:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.259149:0:2406:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.259156:0:2406:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074847.259175:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617022976:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074847.262507:0:2407:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.262534:0:2407:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 6/2,0 mode: --/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.262549:0:2407:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.262561:0:2407:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074847.262578:0:2407:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074847.262597:0:2407:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:1.0:1769074847.262617:0:2407:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.262626:0:2407:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.262633:0:2407:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.262641:0:2407:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074847.262672:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617023104:192.168.204.160@tcp:101:lfs.0 00000080:00200000:0.0:1769074847.263003:0:66757:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x207:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1769074847.263019:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_release():245 00000020:00010000:0.0:1769074847.263022:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074847.263028:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03492104b0 (ffff9b03531522d8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074847.263033:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03492104f0 (ffff9b0342b7d758/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074847.263036:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074847.263039:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03492104b0 (ffff9b0353152a28/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074847.263043:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03492104f0 (ffff9b0353152a28/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074847.263058:0:66757:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 0/1, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074847.263111:0:66757:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074847.263135:0:66757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074847.263149:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.263163:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.263170:0:66757:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074847.263178:0:66757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074847.263182:0:66757:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:0.0:1769074847.263200:0:66757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641c77bf8 00010000:00010000:0.0:1769074847.263203:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.263210:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.263221:0:66757:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074847.263228:0:66757:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=25 00000080:00200000:0.0:1769074847.263762:0:66757:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1769074847.263770:0:66757:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204b.sanity-flr, flags=265 00000080:00200000:0.0:1769074847.263776:0:66757:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988), inode mode 81a4 mask 46 00000080:00200008:0.0:1769074847.263785:0:66757:0:(file.c:1001:ll_file_open()) START file f204b.sanity-flr:[0x200000405:0x207:0x0](ffff9b0355d41988), flags 154402 00000002:00010000:0.0:1769074847.263807:0:66757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x207:0x0]) in obj [0x200000405:0x207:0x0], intent: open flags 0100240114403 00010000:00010000:0.0:1769074847.263879:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 3/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 66757 initiator: MDT0 00010000:00010000:0.0:1769074847.263887:0:66757:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 3/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 66757 initiator: MDT0 00000020:00100000:0.0:1769074847.263899:0:66757:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074847.263912:0:66757:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a38700 x1855007617023360/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074847.263921:0:66757:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 3/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 66757 initiator: MDT0 00000100:00100000:0.0:1769074847.263949:0:66757:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617023360:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074847.264675:0:66757:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a000 going to sleep for 50 seconds 00000100:00100000:1.0:1769074847.269507:0:66757:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617023360:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074847.269535:0:66757:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 4/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.269547:0:66757:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 4/1,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.269558:0:66757:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.269568:0:66757:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.269581:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.269590:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.269597:0:66757:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074847.269619:0:66757:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9b0365a38700 x1855007617023360/t8589944109(8589944109) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/1160 e 0 to 0 dl 1769074902 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074847.269634:0:66757:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0365a38700 x1855007617023360/t8589944109(8589944109) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/1160 e 0 to 0 dl 1769074902 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074847.269649:0:66757:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074847.269673:0:66757:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.269689:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.269702:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.269710:0:66757:0:(ldlm_lock.c:947:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.269723:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.269729:0:66757:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.269752:0:66757:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:1.0:1769074847.269759:0:66757:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000405:0x207:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1769074847.269863:0:66757:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x207:0x0]: layout version change: 6 -> 6 00000080:00010000:1.0:1769074847.269870:0:66757:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c7d7 00000080:00010000:1.0:1769074847.269876:0:66757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c7d7 from it ffffb43641c77bf0 00010000:00010000:1.0:1769074847.269880:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.269888:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000002:00100000:1.0:1769074847.269914:0:66757:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0365a38700 x1855007617023360/t8589944109(8589944109) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074902 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074847.270354:0:66757:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988), inode mode 81a4 mask 4 00000080:00200000:1.0:1769074847.270363:0:66757:0:(xattr.c:725:ll_xattr_get()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988), xattr lov 00000080:00200008:1.0:1769074847.270390:0:66757:0:(file.c:6252:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000405:0x207:0x0](ffff9b0355d41988), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074847.270396:0:66757:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988),name=f204b.sanity-flr 00000002:00010000:1.0:1769074847.270415:0:66757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x207:0x0]) in obj [0x200000bd0:0x8:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074847.270425:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.270438:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074847.270445:0:66757:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x8:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074847.270451:0:66757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) 00000080:00010000:1.0:1769074847.270454:0:66757:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c7d7 00000080:00010000:1.0:1769074847.270458:0:66757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c7d7 from it ffffb43641c77cd8 00010000:00010000:1.0:1769074847.270461:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.270468:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074847.270485:0:66757:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1769074847.270496:0:66757:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x207:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074847.270500:0:66757:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1769074847.270503:0:66757:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x207:0x0] 00000008:00010000:1.0:1769074847.270538:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074847.270542:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0349210db0, osc lock ffff9b0345a09560, flags 0x400000400001002 00000008:00010000:1.0:1769074847.270559:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074847.270560:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0349210df0, osc lock ffff9b0345a09300, flags 0x400000400001002 00010000:00010000:1.0:1769074847.270576:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.270586:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.270693:0:60220:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769074847.270701:0:60220:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074847.270709:0:60220:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0348215400 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074847.270714:0:60220:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074847.270722:0:60220:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074847.270735:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 3/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074847.270751:0:60220:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074847.270786:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 2/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074847.270793:0:60220:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074847.270820:0:60220:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 1/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074847.270828:0:60220:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c7ec lrc: 0/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd556af expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074847.270856:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074847.270858:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074847.270860:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074847.270876:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617023488:192.168.204.160@tcp:103:ldlm_bl.0 00000020:00200000:1.0:1769074847.271412:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074847.271421:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.271431:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.271451:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.271463:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074847.271471:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074847.271474:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.271480:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074847.271487:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074847.271499:0:66757:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x207:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1769074847.271504:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0356961da8 (ffff9b035121f638/0) at cl_lock_release():245 00000020:00010000:1.0:1769074847.271507:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0356961da8 (ffff9b035121f638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074847.271511:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0349210db0 (ffff9b035121f128/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074847.271514:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0349210df0 (ffff9b035121f758/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074847.271517:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0356961da8 (ffff9b035121f638/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074847.271519:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0349210db0 (ffff9b035121f638/1) at cl_lock_fini():72 00000020:00010000:1.0:1769074847.271524:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0349210df0 (ffff9b035121f638/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074847.271536:0:66757:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 0/1, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1769074847.271579:0:66757:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988) cmd=c00c66f3 arg=12992a0 00000002:00010000:1.0:1769074847.272920:0:66757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x207:0x0]) in obj [0x200000405:0x207:0x0], intent: open flags 05004040000002 00010000:00010000:1.0:1769074847.272988:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 66757 initiator: MDT0 00010000:00010000:1.0:1769074847.272997:0:66757:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 66757 initiator: MDT0 00000020:00100000:1.0:1769074847.273102:0:66757:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074847.273116:0:66757:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ceb480 x1855007617023744/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074847.273126:0:66757:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x81000000000000 pid: 66757 initiator: MDT0 00000100:00100000:1.0:1769074847.273151:0:66757:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617023744:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074847.273481:0:66757:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210d80 going to sleep for 50 seconds 00000100:00100000:1.0:1769074847.276632:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617023488:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074847.279596:0:66757:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617023744:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074847.279691:0:66757:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 4/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.279704:0:66757:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 4/1,0 mode: --/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.279716:0:66757:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 4/1,0 mode: PW/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.279732:0:66757:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 4/1,0 mode: PW/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.279745:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 5/1,1 mode: PW/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.279753:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 5/1,1 mode: PW/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.279761:0:66757:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 4/0,1 mode: PW/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074847.279772:0:66757:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=600002b, status=0 req@ffff9b0342ceb480 x1855007617023744/t8589944110(8589944110) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/1160 e 0 to 0 dl 1769074902 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074847.279786:0:66757:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0342ceb480 x1855007617023744/t8589944110(8589944110) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/1160 e 0 to 0 dl 1769074902 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074847.279803:0:66757:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 3/0,1 mode: PW/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.279816:0:66757:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 2 res 8589935621/519 (0 0) 00000080:00200000:1.0:1769074847.279822:0:66757:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000405:0x207:0x0] suppgids 0 0: rc 0 00000002:00100000:1.0:1769074847.279840:0:66757:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0342ceb480 x1855007617023744/t8589944110(8589944110) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074902 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074847.279859:0:66757:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c7f3 00000080:00010000:1.0:1769074847.279866:0:66757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c7f3 from it ffffb43641c77cc0 00010000:00010000:1.0:1769074847.279870:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 3/0,1 mode: PW/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.279878:0:66757:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 2/0,0 mode: PW/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.279945:0:66757:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00020000:00200000:1.0:1769074847.279959:0:66757:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x207:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074847.279963:0:66757:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1769074847.279969:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1769074847.279979:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1769074847.279984:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000100:00100000:0.0:1769074847.280168:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617023872:192.168.204.160@tcp:1:lfs.0 00000100:00100000:0.0:1769074847.281215:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1769074847.281230:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617024000:192.168.204.160@tcp:1:lfs.0 00000100:00100000:0.0:1769074847.284080:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074847.284121:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074847.284153:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625233536 00000100:00100000:0.0:1769074847.284164:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625233536 00000100:00100000:0.0:1769074847.284219:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625233664 00000100:00100000:0.0:1769074847.284222:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625233664 00000100:00100000:0.0:1769074847.284282:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8938 00000100:00100000:0.0:1769074847.284292:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a39880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:28955:x1855007625233536:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074847.284309:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164c800/0xb7d6bf619879c7de lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd5569a expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.284373:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074847.284379:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a39880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:28955:x1855007625233536:12345-192.168.204.160@tcp:104: Request processed in 90us (315us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074847.284387:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8938 00000100:00100000:0.0:1769074847.284405:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8939 00000100:00100000:0.0:1769074847.284410:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a3b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:9048:x1855007625233664:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074847.284419:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b034164d800/0xb7d6bf619879c7e5 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd556a1 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.284456:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074847.284462:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a3b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:9048:x1855007625233664:12345-192.168.204.160@tcp:104: Request processed in 51us (340us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074847.284467:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8939 00000100:00100000:0.0:1769074847.290620:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617024000:192.168.204.160@tcp:1:lfs.0 00000100:00100000:0.0:1769074847.290641:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617023872:192.168.204.160@tcp:1:lfs.0 00010000:00010000:1.0:1769074847.291865:0:66757:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074847.291893:0:66757:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x207:0x0]: losing layout lock 00010000:00010000:1.0:1769074847.291915:0:66757:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c798 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55566 expref: -99 pid: 66727 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074847.291924:0:66757:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x207:0x0]: losing layout lock 00010000:00010000:1.0:1769074847.291962:0:66757:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.291970:0:66757:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c798 lrc: 1/0,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55566 expref: -99 pid: 66727 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.291977:0:66757:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769074847.291979:0:66757:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164cc00/0xb7d6bf619879c7d7 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55693 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074847.291994:0:66757:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c798 lrc: 0/0,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55566 expref: -99 pid: 66727 timeout: 0 lvb_type: 3 00000020:00100000:1.0:1769074847.292135:0:66757:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074847.292147:0:66757:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ceb800 x1855007617024512/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 472/440 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074847.292179:0:66757:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617024512:192.168.204.160@tcp:36:lfs.0 00000100:00100000:0.0:1769074847.293061:0:66757:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210d80 going to sleep for 50 seconds 00000008:00010000:2.0:1769074847.480931:0:2409:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b0342c54e40 00000100:00100000:2.0:1769074847.480968:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617025792:192.168.204.160@tcp:3:lfs.0 00000008:00010000:2.0:1769074847.487287:0:2409:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b0342c54cc0 00000020:00200000:2.0:1769074847.487385:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1769074847.487391:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1769074847.487393:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1769074847.487395:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1769074847.487396:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1769074847.487398:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1769074847.487400:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1769074847.487401:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1769074847.487402:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1769074847.487404:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1769074847.487405:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1769074847.487407:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1769074847.487408:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1769074847.487410:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1769074847.487411:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1769074847.487413:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1769074847.487414:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1769074847.487416:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1769074847.487417:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1769074847.487418:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1769074847.487420:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1769074847.487421:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1769074847.487422:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1769074847.487423:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1769074847.487425:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1769074847.487427:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1769074847.487428:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1769074847.487430:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1769074847.487431:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1769074847.487433:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1769074847.487434:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1769074847.487435:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1769074847.487437:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1769074847.487438:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1769074847.487440:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1769074847.487441:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1769074847.487443:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1769074847.487444:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1769074847.487446:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1769074847.487447:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1769074847.487448:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1769074847.487450:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1769074847.487452:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1769074847.487453:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1769074847.487455:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1769074847.487456:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1769074847.487458:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1769074847.487459:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1769074847.487461:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1769074847.487462:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1769074847.487464:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1769074847.487466:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1769074847.487468:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1769074847.487469:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1769074847.487471:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1769074847.487472:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1769074847.487474:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1769074847.487475:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1769074847.487476:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1769074847.487478:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1769074847.487479:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1769074847.487481:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1769074847.487483:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1769074847.487485:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1769074847.487486:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1769074847.487488:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1769074847.487489:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1769074847.487490:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1769074847.487492:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1769074847.487493:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1769074847.487495:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1769074847.487497:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1769074847.487498:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1769074847.487500:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1769074847.487501:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1769074847.487503:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1769074847.487504:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1769074847.487506:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1769074847.487507:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1769074847.487508:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1769074847.487510:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1769074847.487511:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1769074847.487512:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1769074847.487514:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1769074847.487515:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1769074847.487516:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1769074847.487518:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1769074847.487519:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1769074847.487521:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1769074847.487522:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1769074847.487524:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1769074847.487525:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1769074847.487527:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1769074847.487529:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1769074847.487531:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1769074847.487532:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1769074847.487534:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1769074847.487535:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1769074847.487536:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1769074847.487538:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1769074847.487539:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1769074847.487540:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1769074847.487542:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1769074847.487543:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1769074847.487544:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1769074847.487546:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1769074847.487547:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1769074847.487549:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1769074847.487550:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1769074847.487552:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1769074847.487553:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1769074847.487554:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1769074847.487556:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1769074847.487558:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1769074847.487559:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1769074847.487561:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1769074847.487562:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1769074847.487564:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1769074847.487566:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1769074847.487567:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1769074847.487568:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1769074847.487570:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1769074847.487571:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1769074847.487572:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1769074847.487574:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1769074847.487576:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1769074847.487577:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1769074847.487579:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1769074847.487580:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1769074847.487582:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1769074847.487583:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1769074847.487584:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1769074847.487585:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1769074847.487587:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1769074847.487588:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1769074847.487589:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1769074847.487591:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1769074847.487593:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1769074847.487595:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1769074847.487596:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1769074847.487598:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1769074847.487600:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1769074847.487602:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1769074847.487604:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1769074847.487605:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1769074847.487607:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1769074847.487609:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1769074847.487610:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1769074847.487612:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1769074847.487613:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1769074847.487615:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1769074847.487616:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1769074847.487618:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1769074847.487620:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1769074847.487622:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1769074847.487623:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1769074847.487625:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1769074847.487628:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1769074847.487629:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1769074847.487631:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1769074847.487633:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1769074847.487634:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1769074847.487636:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1769074847.487638:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1769074847.487639:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1769074847.487640:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1769074847.487642:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1769074847.487643:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1769074847.487644:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1769074847.487646:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1769074847.487648:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1769074847.487649:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1769074847.487651:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1769074847.487654:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1769074847.487656:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1769074847.487657:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1769074847.487659:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1769074847.487660:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1769074847.487662:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1769074847.487663:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1769074847.487665:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1769074847.487667:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1769074847.487668:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1769074847.487670:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1769074847.487672:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1769074847.487673:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1769074847.487675:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1769074847.487676:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1769074847.487678:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1769074847.487679:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1769074847.487681:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1769074847.487683:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1769074847.487684:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1769074847.487685:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1769074847.487687:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1769074847.487688:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1769074847.487689:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1769074847.487691:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1769074847.487692:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1769074847.487694:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1769074847.487695:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1769074847.487697:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1769074847.487699:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1769074847.487701:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1769074847.487702:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1769074847.487704:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1769074847.487706:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1769074847.487707:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1769074847.487709:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1769074847.487711:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1769074847.487712:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1769074847.487714:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1769074847.487716:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1769074847.487717:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1769074847.491484:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1769074847.491490:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1769074847.491492:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1769074847.491494:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1769074847.491496:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1769074847.491498:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1769074847.491499:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1769074847.491502:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1769074847.491504:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1769074847.491505:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1769074847.491507:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1769074847.491509:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1769074847.491511:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1769074847.491512:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1769074847.491514:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1769074847.491515:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1769074847.491517:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1769074847.491519:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1769074847.491520:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1769074847.491522:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1769074847.491523:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1769074847.491525:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1769074847.491526:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1769074847.491528:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1769074847.491529:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1769074847.491531:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1769074847.491533:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1769074847.491534:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1769074847.491536:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1769074847.491538:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1769074847.491539:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1769074847.491541:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1769074847.491542:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1769074847.491545:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1769074847.491546:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1769074847.491548:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1769074847.491549:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1769074847.491551:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1769074847.491553:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1769074847.491555:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1769074847.491557:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1769074847.491558:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1769074847.491681:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1769074847.491709:0:2409:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b0342c54cc0 00000100:00100000:2.0:1769074847.491747:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617025664:192.168.204.160@tcp:3:lfs.0 00000100:00100000:2.0:1769074847.498579:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617025920:192.168.204.160@tcp:4:lfs.0 00000100:00100000:2.0:1769074847.556939:0:2408:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9b0342ad2800: committing for last_committed 17179874627 gen 2 00000008:00010000:2.0:1769074847.557124:0:2408:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b035349f0c0 00000020:00200000:2.0:1769074847.557132:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:2.0:1769074847.557135:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:2.0:1769074847.557137:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:2.0:1769074847.557139:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:2.0:1769074847.557141:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:2.0:1769074847.557142:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:2.0:1769074847.557144:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:2.0:1769074847.557146:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:2.0:1769074847.557147:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:2.0:1769074847.557149:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:2.0:1769074847.557150:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:2.0:1769074847.557152:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:2.0:1769074847.557153:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:2.0:1769074847.557155:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:2.0:1769074847.557156:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:2.0:1769074847.557158:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:2.0:1769074847.557160:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:2.0:1769074847.557161:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:2.0:1769074847.557162:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:2.0:1769074847.557164:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:2.0:1769074847.557165:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:2.0:1769074847.557168:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:2.0:1769074847.557169:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:2.0:1769074847.557171:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:2.0:1769074847.557173:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:2.0:1769074847.557174:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:2.0:1769074847.557176:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:2.0:1769074847.557177:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:2.0:1769074847.557179:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:2.0:1769074847.557180:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:2.0:1769074847.557182:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:2.0:1769074847.557183:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:2.0:1769074847.557185:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:2.0:1769074847.557308:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:2.0:1769074847.557309:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:2.0:1769074847.557311:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:2.0:1769074847.557313:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:2.0:1769074847.557314:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:2.0:1769074847.557316:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:2.0:1769074847.557318:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:2.0:1769074847.557319:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:2.0:1769074847.557321:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:2.0:1769074847.557322:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:2.0:1769074847.557324:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:2.0:1769074847.557325:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:2.0:1769074847.557327:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:2.0:1769074847.557328:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:2.0:1769074847.557330:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:2.0:1769074847.557331:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:2.0:1769074847.557332:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:2.0:1769074847.557334:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:2.0:1769074847.557335:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:2.0:1769074847.557337:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:2.0:1769074847.557338:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:2.0:1769074847.557340:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:2.0:1769074847.557342:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:2.0:1769074847.557344:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:2.0:1769074847.557345:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:2.0:1769074847.557347:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:2.0:1769074847.557348:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:2.0:1769074847.557350:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:2.0:1769074847.557352:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:2.0:1769074847.557353:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:2.0:1769074847.557355:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:2.0:1769074847.557357:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:2.0:1769074847.557359:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:2.0:1769074847.557360:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:2.0:1769074847.557362:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:2.0:1769074847.557364:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:2.0:1769074847.557366:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:2.0:1769074847.557367:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:2.0:1769074847.557369:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:2.0:1769074847.557371:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:2.0:1769074847.557373:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:2.0:1769074847.557374:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:2.0:1769074847.557375:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:2.0:1769074847.557377:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:2.0:1769074847.557379:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:2.0:1769074847.557381:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:2.0:1769074847.557382:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:2.0:1769074847.557384:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:2.0:1769074847.557385:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:2.0:1769074847.557387:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:2.0:1769074847.557388:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:2.0:1769074847.557390:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:2.0:1769074847.557392:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:2.0:1769074847.557393:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:2.0:1769074847.557395:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:2.0:1769074847.557397:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:2.0:1769074847.557399:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:2.0:1769074847.557400:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:2.0:1769074847.557402:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:2.0:1769074847.557403:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:2.0:1769074847.557405:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:2.0:1769074847.557406:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:2.0:1769074847.557408:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:2.0:1769074847.557409:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:2.0:1769074847.557410:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:2.0:1769074847.557412:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:2.0:1769074847.557413:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:2.0:1769074847.557415:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:2.0:1769074847.557417:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:2.0:1769074847.557418:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:2.0:1769074847.557420:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:2.0:1769074847.557422:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:2.0:1769074847.557424:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:2.0:1769074847.557425:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:2.0:1769074847.557427:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:2.0:1769074847.557428:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:2.0:1769074847.557430:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:2.0:1769074847.557431:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:2.0:1769074847.557433:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:2.0:1769074847.557434:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:2.0:1769074847.557436:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:2.0:1769074847.557437:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:2.0:1769074847.557439:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:2.0:1769074847.557440:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:2.0:1769074847.557442:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:2.0:1769074847.557444:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:2.0:1769074847.557445:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:2.0:1769074847.557447:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:2.0:1769074847.557448:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:2.0:1769074847.557449:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:2.0:1769074847.557451:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:2.0:1769074847.557453:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:2.0:1769074847.557454:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:2.0:1769074847.557456:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:2.0:1769074847.557457:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:2.0:1769074847.557459:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:2.0:1769074847.557460:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:2.0:1769074847.557462:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:2.0:1769074847.557464:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:2.0:1769074847.557465:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:2.0:1769074847.557467:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:2.0:1769074847.557468:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:2.0:1769074847.557470:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:2.0:1769074847.557471:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:2.0:1769074847.557473:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:2.0:1769074847.557475:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:2.0:1769074847.557477:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:2.0:1769074847.557478:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:2.0:1769074847.557480:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:2.0:1769074847.557482:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:2.0:1769074847.557483:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:2.0:1769074847.557485:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:2.0:1769074847.557486:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:2.0:1769074847.557488:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:2.0:1769074847.557490:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:2.0:1769074847.557491:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:2.0:1769074847.557493:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:2.0:1769074847.557495:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:2.0:1769074847.557497:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:2.0:1769074847.557498:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:2.0:1769074847.557500:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:2.0:1769074847.557501:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:2.0:1769074847.557503:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:2.0:1769074847.557504:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:2.0:1769074847.557506:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:2.0:1769074847.557508:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:2.0:1769074847.557509:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:2.0:1769074847.557511:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:2.0:1769074847.557513:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:2.0:1769074847.557515:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:2.0:1769074847.557516:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:2.0:1769074847.557518:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:2.0:1769074847.557598:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:2.0:1769074847.557600:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:2.0:1769074847.557601:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:2.0:1769074847.557603:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:2.0:1769074847.557605:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:2.0:1769074847.557606:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:2.0:1769074847.557608:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:2.0:1769074847.557610:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:2.0:1769074847.557611:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:2.0:1769074847.557613:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:2.0:1769074847.557615:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:2.0:1769074847.557616:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:2.0:1769074847.557618:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:2.0:1769074847.557620:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:2.0:1769074847.557621:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:2.0:1769074847.557623:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:2.0:1769074847.557625:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:2.0:1769074847.557627:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:2.0:1769074847.557629:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:2.0:1769074847.557630:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:2.0:1769074847.557632:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:2.0:1769074847.557634:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:2.0:1769074847.557636:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:2.0:1769074847.557637:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:2.0:1769074847.557639:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:2.0:1769074847.557641:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:2.0:1769074847.557642:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:2.0:1769074847.557644:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:2.0:1769074847.557646:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:2.0:1769074847.557648:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:2.0:1769074847.557650:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:2.0:1769074847.557651:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:2.0:1769074847.557653:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:2.0:1769074847.557655:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:2.0:1769074847.557731:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:2.0:1769074847.557734:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:2.0:1769074847.557736:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:2.0:1769074847.557737:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:2.0:1769074847.557739:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:2.0:1769074847.557741:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:2.0:1769074847.557743:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:2.0:1769074847.557745:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:2.0:1769074847.557746:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:2.0:1769074847.557748:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:2.0:1769074847.557750:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:2.0:1769074847.557751:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:2.0:1769074847.557753:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:2.0:1769074847.557755:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:2.0:1769074847.557757:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:2.0:1769074847.557758:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:2.0:1769074847.557760:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:2.0:1769074847.557762:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:2.0:1769074847.557763:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:2.0:1769074847.557765:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:2.0:1769074847.557766:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:2.0:1769074847.557768:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:2.0:1769074847.557769:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:2.0:1769074847.557771:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:2.0:1769074847.557772:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:2.0:1769074847.557774:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:2.0:1769074847.557776:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:2.0:1769074847.557777:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:2.0:1769074847.557779:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:2.0:1769074847.557850:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:2.0:1769074847.557852:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:2.0:1769074847.557854:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:2.0:1769074847.557855:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:2.0:1769074847.557857:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:2.0:1769074847.557859:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:2.0:1769074847.557860:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:2.0:1769074847.557862:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:2.0:1769074847.557863:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:2.0:1769074847.557865:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:2.0:1769074847.557867:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:2.0:1769074847.557868:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:2.0:1769074847.557870:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:2.0:1769074847.557871:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:2.0:1769074847.557873:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:2.0:1769074847.557875:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:2.0:1769074847.557876:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:2.0:1769074847.557878:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:2.0:1769074847.557880:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:2.0:1769074847.557881:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:2.0:1769074847.557883:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:2.0:1769074847.557884:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:2.0:1769074847.557886:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:2.0:1769074847.557887:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:2.0:1769074847.557889:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:2.0:1769074847.557891:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:2.0:1769074847.557892:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:2.0:1769074847.557894:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:2.0:1769074847.557895:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1769074847.557897:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1769074847.557899:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1769074847.557900:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1769074847.557902:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1769074847.557904:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1769074847.557906:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1769074847.557907:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1769074847.557909:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1769074847.557911:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1769074847.557912:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1769074847.557914:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1769074847.557915:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1769074847.557917:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1769074847.557918:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1769074847.557920:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1769074847.557922:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1769074847.557923:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1769074847.557925:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1769074847.557927:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1769074847.557928:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1769074847.557930:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1769074847.557932:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1769074847.557934:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1769074847.557936:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1769074847.557938:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1769074847.557939:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1769074847.557941:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1769074847.557942:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1769074847.557944:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1769074847.557945:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1769074847.557947:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1769074847.557949:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1769074847.557950:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1769074847.557952:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1769074847.557954:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1769074847.557955:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1769074847.557957:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1769074847.557958:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1769074847.557960:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1769074847.557961:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1769074847.557963:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1769074847.557965:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1769074847.557966:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1769074847.557968:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1769074847.557969:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1769074847.557971:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1769074847.557972:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1769074847.557974:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1769074847.557975:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1769074847.557977:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1769074847.557978:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1769074847.557980:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1769074847.557982:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1769074847.557984:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1769074847.557986:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1769074847.557987:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1769074847.557989:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1769074847.557991:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1769074847.557993:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1769074847.557994:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1769074847.557996:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1769074847.557997:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1769074847.557999:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1769074847.558090:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1769074847.558094:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1769074847.558095:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1769074847.558097:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1769074847.558098:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1769074847.558099:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1769074847.558101:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1769074847.558102:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1769074847.558104:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1769074847.558105:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1769074847.558107:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1769074847.558108:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1769074847.558110:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1769074847.558112:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1769074847.558113:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1769074847.558115:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1769074847.558117:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1769074847.558118:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1769074847.558120:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1769074847.558121:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1769074847.558123:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1769074847.558125:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1769074847.558127:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1769074847.558129:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1769074847.558131:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1769074847.558133:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1769074847.558134:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1769074847.558136:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1769074847.558138:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1769074847.558140:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1769074847.558141:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1769074847.558143:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1769074847.558144:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1769074847.558146:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1769074847.558148:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1769074847.558149:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1769074847.558151:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1769074847.558152:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1769074847.558154:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1769074847.558156:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1769074847.558158:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1769074847.558160:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1769074847.558161:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1769074847.558163:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1769074847.558164:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1769074847.558166:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1769074847.558167:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1769074847.558168:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1769074847.558170:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1769074847.558171:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1769074847.558173:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1769074847.558175:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1769074847.558176:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1769074847.558178:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1769074847.558180:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1769074847.558181:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1769074847.558183:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1769074847.558184:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1769074847.558372:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1769074847.558375:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1769074847.558377:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1769074847.558378:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1769074847.558380:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1769074847.558381:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1769074847.558383:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1769074847.558385:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1769074847.558386:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1769074847.558388:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1769074847.558390:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1769074847.558391:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1769074847.558393:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1769074847.558395:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1769074847.558397:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1769074847.558399:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1769074847.558400:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1769074847.558402:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1769074847.558404:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1769074847.558406:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1769074847.558408:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1769074847.558410:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1769074847.558411:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1769074847.558413:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1769074847.558415:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1769074847.558416:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1769074847.558418:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1769074847.558421:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1769074847.558423:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1769074847.558425:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1769074847.558426:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1769074847.558428:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1769074847.558430:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1769074847.558432:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1769074847.558434:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1769074847.558436:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1769074847.558438:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1769074847.558440:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1769074847.558442:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1769074847.558444:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1769074847.558445:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1769074847.558447:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1769074847.558448:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1769074847.558450:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1769074847.558451:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1769074847.558453:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1769074847.558454:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1769074847.558456:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1769074847.558458:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1769074847.558459:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1769074847.558461:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1769074847.558462:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1769074847.558463:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1769074847.558465:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1769074847.558467:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1769074847.558469:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1769074847.558470:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1769074847.558472:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1769074847.558473:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1769074847.558475:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1769074847.558477:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1769074847.558479:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1769074847.558480:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1769074847.558482:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1769074847.558484:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1769074847.558486:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1769074847.558487:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1769074847.558488:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1769074847.558490:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1769074847.558491:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1769074847.558493:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1769074847.558495:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1769074847.558497:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1769074847.558498:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1769074847.558500:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1769074847.558502:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1769074847.558503:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1769074847.558505:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1769074847.558506:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1769074847.558508:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1769074847.558510:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1769074847.558511:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1769074847.558513:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1769074847.558514:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1769074847.558516:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1769074847.558517:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1769074847.558592:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1769074847.558594:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1769074847.558596:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1769074847.558597:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1769074847.558599:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1769074847.558601:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1769074847.558603:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1769074847.558605:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1769074847.558607:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1769074847.558608:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1769074847.558610:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1769074847.558612:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1769074847.558613:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1769074847.558615:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1769074847.558617:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1769074847.558618:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1769074847.558620:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1769074847.558621:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1769074847.558623:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1769074847.558624:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1769074847.558625:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1769074847.558627:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1769074847.558629:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1769074847.558631:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1769074847.558632:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1769074847.558634:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1769074847.558636:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1769074847.558637:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1769074847.558639:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1769074847.558641:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1769074847.558642:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1769074847.558644:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1769074847.558645:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1769074847.558647:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1769074847.558648:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1769074847.558650:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1769074847.558651:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1769074847.558653:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1769074847.558655:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1769074847.558728:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1769074847.558730:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1769074847.558732:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1769074847.558734:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1769074847.558735:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1769074847.558737:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1769074847.558739:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1769074847.558741:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1769074847.558742:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1769074847.559420:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1769074847.559511:0:2408:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b035349f0c0 00000100:00100000:2.0:1769074847.559551:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617025920:192.168.204.160@tcp:4:lfs.0 00000080:00200000:2.0:1769074847.560744:0:66757:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x207:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:2.0:1769074847.560750:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074847.560761:0:66757:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 2097152 00000080:00200080:2.0:1769074847.560778:0:66757:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988) cmd=400866b1 arg=0 00000080:00200080:2.0:1769074847.560786:0:66757:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988) cmd=400866b1 arg=1 00000080:00200008:2.0:1769074847.560793:0:66757:0:(file.c:6252:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000405:0x207:0x0](ffff9b0355d41988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074847.560800:0:66757:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988),name=f204b.sanity-flr 00000002:00010000:2.0:1769074847.560825:0:66757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x207:0x0]) in obj [0x200000405:0x207:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074847.560853:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.560869:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074847.560878:0:66757:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000405:0x207:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074847.560886:0:66757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) 00000080:00010000:2.0:1769074847.560890:0:66757:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c801 00000080:00010000:2.0:1769074847.560897:0:66757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c801 from it ffffb43641c77cd8 00010000:00010000:2.0:1769074847.560901:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.560910:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074847.560933:0:66757:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074847.560943:0:66757:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x207:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074847.560946:0:66757:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074847.560950:0:66757:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x207:0x0] 00000008:00010000:2.0:1769074847.560969:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074847.560972:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360b5ee70, osc lock ffff9b03451265f0, flags 0x400000400001002 00000008:00010000:2.0:1769074847.560979:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074847.560981:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360b5eeb0, osc lock ffff9b0345126980, flags 0x400000400001002 00010000:00010000:2.0:1769074847.560997:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561101:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074847.561112:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074847.561116:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561124:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561144:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561153:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074847.561160:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074847.561163:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561172:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074847.561180:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074847.561370:0:66757:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x207:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1769074847.561375:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314fb50 (ffff9b0353152518/0) at cl_lock_release():245 00000020:00010000:2.0:1769074847.561379:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314fb50 (ffff9b0353152518/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.561382:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360b5ee70 (ffff9b03531521b8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.561386:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360b5eeb0 (ffff9b0353152e18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.561389:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314fb50 (ffff9b0353152518/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074847.561391:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360b5ee70 (ffff9b0353152518/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074847.561395:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360b5eeb0 (ffff9b0353152518/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074847.561404:0:66757:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074847.561416:0:66757:0:(file.c:5185:ll_file_seek()) START file f204b.sanity-flr:[0x200000405:0x207:0x0], offset: 2097152, type: SEEK_DATA 00000080:00200000:2.0:1769074847.561422:0:66757:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:2.0:1769074847.561427:0:66757:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x207:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074847.561429:0:66757:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074847.561434:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074847.561439:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074847.561451:0:66757:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x207:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:2.0:1769074847.561459:0:66757:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074847.561463:0:66757:0:(file.c:1759:ll_io_set_mirror()) f204b.sanity-flr: desiginated mirror: 1 00000080:00200000:2.0:1769074847.561465:0:66757:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074847.561469:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074847.561471:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074847.561474:0:66757:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [512, 18446744073709551615] 00000008:00010000:2.0:1769074847.561481:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1769074847.561483:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348d6b530, osc lock ffff9b0345126980, flags 0x0 00010000:00010000:2.0:1769074847.561491:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561500:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (2097152 68157439) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561510:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561518:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561603:0:66757:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074847.561615:0:66757:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x207:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1769074847.561619:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0343c7e3f8 (ffff9b0353152518/0) at cl_lock_release():245 00000020:00010000:2.0:1769074847.561622:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343c7e3f8 (ffff9b0353152518/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.561625:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6b530 (ffff9b0353152e18/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074847.561629:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561637:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074847.561645:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343c7e3f8 (ffff9b0353152518/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074847.561648:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6b530 (ffff9b0353152518/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074847.561655:0:66757:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074847.561735:0:66757:0:(file.c:5185:ll_file_seek()) START file f204b.sanity-flr:[0x200000405:0x207:0x0], offset: 0, type: (null) 00000080:00200000:2.0:1769074847.561739:0:66757:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074847.561744:0:66757:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x207:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074847.561747:0:66757:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074847.561749:0:66757:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x207:0x0] 00000008:00010000:2.0:1769074847.561757:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074847.561760:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360b5ee70, osc lock ffff9b0345126980, flags 0x400000400001002 00000008:00010000:2.0:1769074847.561764:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074847.561765:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360b5eeb0, osc lock ffff9b03451265f0, flags 0x400000400001002 00010000:00010000:2.0:1769074847.561773:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561782:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074847.561789:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074847.561792:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561799:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561808:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561815:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074847.561821:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074847.561823:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.561830:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074847.561837:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074847.561852:0:66757:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x207:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1769074847.561855:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314fb50 (ffff9b0353152518/0) at cl_lock_release():245 00000020:00010000:2.0:1769074847.561858:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314fb50 (ffff9b0353152518/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.561860:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360b5ee70 (ffff9b0353152e18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.561863:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360b5eeb0 (ffff9b03531521b8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.561866:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314fb50 (ffff9b0353152518/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074847.561868:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360b5ee70 (ffff9b0353152518/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074847.561871:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360b5eeb0 (ffff9b0353152518/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074847.561878:0:66757:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074847.561884:0:66757:0:(file.c:5216:ll_file_seek()) COMPLETED file f204b.sanity-flr:[0x200000405:0x207:0x0], offset: 0, type: (null), rc = 2097152 00000080:00200080:2.0:1769074847.561890:0:66757:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988) cmd=400866b1 arg=0 00000080:00200080:2.0:1769074847.584805:0:66757:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988) cmd=400866b1 arg=2 00000080:00200008:2.0:1769074847.584828:0:66757:0:(file.c:6252:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000405:0x207:0x0](ffff9b0355d41988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074847.584838:0:66757:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988),name=f204b.sanity-flr 00000002:00010000:2.0:1769074847.584875:0:66757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x207:0x0]) in obj [0x200000405:0x207:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074847.584895:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.584911:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074847.584920:0:66757:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000405:0x207:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074847.584928:0:66757:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) 00000080:00010000:2.0:1769074847.584931:0:66757:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c801 00000080:00010000:2.0:1769074847.584938:0:66757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c801 from it ffffb43641c77cd8 00010000:00010000:2.0:1769074847.584942:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.584951:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074847.584980:0:66757:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074847.584993:0:66757:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x207:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074847.584996:0:66757:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074847.585312:0:66757:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x207:0x0] 00000008:00010000:2.0:1769074847.585337:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074847.585341:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360b5ee70, osc lock ffff9b03451265f0, flags 0x400000400001002 00000008:00010000:2.0:1769074847.585349:0:66757:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074847.585351:0:66757:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360b5eeb0, osc lock ffff9b0345126980, flags 0x400000400001002 00010000:00010000:2.0:1769074847.585368:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.585379:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 1048575) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074847.585391:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074847.585396:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.585403:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.585417:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.585426:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (1048576 68157439) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074847.585433:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074847.585436:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.585443:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074847.585455:0:66757:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074847.585465:0:66757:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x207:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1769074847.585470:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314fb50 (ffff9b0353152518/0) at cl_lock_release():245 00000020:00010000:2.0:1769074847.585473:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314fb50 (ffff9b0353152518/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.585476:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360b5ee70 (ffff9b03531521b8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.585479:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360b5eeb0 (ffff9b0353152e18/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.585482:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314fb50 (ffff9b0353152518/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074847.585484:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360b5ee70 (ffff9b0353152518/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074847.585488:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360b5eeb0 (ffff9b0353152518/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074847.585500:0:66757:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074847.585530:0:66757:0:(llite_lib.c:2314:ll_setattr_raw()) START file f204b.sanity-flr:[0x200000405:0x207:0x0](ffff9b0355d41988) current size 2097152, valid attrs 2068, mode 0, uid -1802657998, gid -1, new size 2097152, atime 1769074847.000000000, mtime 1769074847.000000000, ctime 1769074847.000000000 00010000:00010000:2.0:1769074847.585553:0:66757:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.585568:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c7fa lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd556e7 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.585611:0:66757:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.585618:0:66757:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074847.585621:0:66757:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c801 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd556f5 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000020:00100000:2.0:1769074847.585641:0:66757:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074847.585654:0:66757:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c5180 x1855007617026176/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074847.585678:0:66757:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617026176:192.168.204.160@tcp:36:lfs.0 00000100:00100000:2.0:1769074847.585794:0:66757:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5ee40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074847.602724:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617026560:192.168.204.160@tcp:10:lfs.0 00000100:00100000:2.0:1769074847.602829:0:2409:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074847.602837:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617026688:192.168.204.160@tcp:10:lfs.0 00000100:00100000:2.0:1769074847.608468:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617026688:192.168.204.160@tcp:10:lfs.0 00000100:00100000:2.0:1769074847.608498:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617026560:192.168.204.160@tcp:10:lfs.0 00000100:00100000:1.0:1769074847.616933:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074847.617048:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625234304 00000100:00100000:0.0:1769074847.617060:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625234304 00000100:00100000:0.0:1769074847.617118:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8940 00000100:00100000:0.0:1769074847.617129:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:27843:x1855007625234304:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074847.617144:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c808 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd556fc expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.617274:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074847.617281:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:27843:x1855007625234304:12345-192.168.204.160@tcp:104: Request processed in 154us (359us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074847.617288:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8940 00000100:00100000:2.0:1769074847.617653:0:2408:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074847.617679:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617027200:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074847.620273:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074847.621282:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625234432 00000100:00100000:0.0:1769074847.621296:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625234432 00000100:00100000:0.0:1769074847.621367:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8941 00000100:00100000:0.0:1769074847.621378:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cea300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:36300:x1855007625234432:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074847.621397:0:39329:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.621410:0:39329:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.621419:0:39329:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.621429:0:39329:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.621439:0:39329:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.621449:0:39329:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x80000000000 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.621469:0:39329:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b0342cd0200) 00000100:00100000:0.0:1769074847.621532:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cea300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:36300:x1855007625234432:12345-192.168.204.160@tcp:105: Request processed in 155us (1307us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074847.621539:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8941 00000100:00100000:2.0:1769074847.628208:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617027200:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074847.630674:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074847.630885:0:39330:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074847.630903:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625234560 00000100:00100000:1.0:1769074847.630912:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625234560 00000100:00100000:1.0:1769074847.630964:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8942 00000100:00100000:1.0:1769074847.630973:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0350463050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:36516:x1855007625234560:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074847.630989:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074847.631077:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074847.631083:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0350463050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:36516:x1855007625234560:12345-192.168.204.160@tcp:104: Request processed in 112us (420us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074847.631090:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8942 00010000:00010000:2.0:1769074847.631426:0:39339:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074847.631432:0:39339:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.631441:0:39339:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0342cd0c00 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:2.0:1769074847.631446:0:39339:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800428400000000 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074847.631465:0:39339:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074847.631480:0:39339:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769074847.631522:0:39339:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 2/0,0 mode: --/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.631533:0:39339:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074847.631568:0:39339:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 1/0,0 mode: --/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.631576:0:39339:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0c00/0xb7d6bf619879c80f lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55703 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.631593:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074847.631595:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074847.631597:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074847.636711:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074847.636859:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625234688 00000100:00100000:0.0:1769074847.636870:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625234688 00000100:00100000:0.0:1769074847.636927:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8943 00000100:00100000:0.0:1769074847.636938:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:36300:x1855007625234688:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074847.636955:0:39329:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.636967:0:39329:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [1048576->68157439] (req 1048576->68157439) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.636977:0:39329:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.636989:0:39329:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.636999:0:39329:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.637102:0:39329:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x80000000000 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074847.637122:0:39329:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b0342cd1e00) 00000100:00100000:0.0:1769074847.637181:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:36300:x1855007625234688:12345-192.168.204.160@tcp:105: Request processed in 245us (479us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074847.637403:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8943 00000020:00010000:2.0:1769074847.646045:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03590b07b8 (ffff9b0342b7d368/0) at cl_lock_release():245 00000020:00010000:2.0:1769074847.646050:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03590b07b8 (ffff9b0342b7d368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074847.646055:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352ad21b0 (ffff9b0342b7d3f8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074847.646065:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.646079:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074847.646096:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352ad21f0 (ffff9b0342b7de18/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074847.646100:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074847.646108:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074847.646115:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03590b07b8 (ffff9b0342b7d368/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074847.646118:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352ad21b0 (ffff9b0342b7d368/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074847.646125:0:66757:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352ad21f0 (ffff9b0342b7d368/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074847.646145:0:66757:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074847.646158:0:66757:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file f204b.sanity-flr:[0x200000405:0x207:0x0](ffff9b0355d41988) current size 2097152, valid attrs 101f0, mode b436, uid 1614805320, gid -25853, new size 18446633037360011656, atime 1769074846.000000000, mtime 1769074846.000000000, ctime 1769074847.000000000, rc 0 00000080:00200080:2.0:1769074847.646182:0:66757:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988) cmd=c00c66f3 arg=12992a0 00010000:00010000:2.0:1769074847.646294:0:66757:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 2/0,0 mode: PW/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.646310:0:66757:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164dc00/0xb7d6bf619879c7f3 lrc: 0/0,0 mode: --/PW res: [0x200000405:0x207:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7fe2323adfd556c4 expref: -99 pid: 66757 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074847.646330:0:66757:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0342ceb480 x1855007617023744/t8589944110(8589944110) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074902 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074847.646340:0:66757:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074847.646345:0:66757:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c4380 x1855007617027968/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 504/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074847.646367:0:66757:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617027968:192.168.204.160@tcp:35:lfs.0 00000100:00100000:2.0:1769074847.646446:0:66757:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5ee40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074847.651878:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074847.652140:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625234816 00000100:00100000:0.0:1769074847.652152:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625234816 00000100:00100000:0.0:1769074847.652216:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8944 00000100:00100000:0.0:1769074847.652226:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6515:x1855007625234816:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074847.652276:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c7fa lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd556e7 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074847.652348:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074847.652355:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6515:x1855007625234816:12345-192.168.204.160@tcp:104: Request processed in 131us (489us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074847.652362:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8944 00010000:00010000:2.0:1769074847.652454:0:60112:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074847.652462:0:60112:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c7fa lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd556e7 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.652470:0:60112:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860a00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074847.652476:0:60112:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c7fa lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7fe2323adfd556e7 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.652487:0:60112:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c7fa lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7fe2323adfd556e7 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.652495:0:60112:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c7fa lrc: 3/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7fe2323adfd556e7 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074847.652519:0:60112:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x207:0x0]: losing layout lock 00010000:00010000:2.0:1769074847.652560:0:60112:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c7fa lrc: 2/0,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd556e7 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.652567:0:60112:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074847.652585:0:60112:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c7fa lrc: 1/0,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd556e7 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.652592:0:60112:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c7fa lrc: 0/0,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd556e7 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.652607:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074847.652608:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074847.652610:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074847.669205:0:66757:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944115 gen 1 00000100:00100000:2.0:1769074847.669275:0:66757:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617027968:192.168.204.160@tcp:35:lfs.0 00000002:00100000:2.0:1769074847.669312:0:66757:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0342ceb480 x1855007617023744/t8589944110(8589944110) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074902 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1769074847.669346:0:66757:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 31 res 8589935621/519 (0 0) 00000080:00010000:2.0:1769074847.669357:0:66757:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000405:0x207:0x0](ffff9b0355d41988) 00000002:00010000:2.0:1769074847.669369:0:66757:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x207:0x0]) in obj [0x200000405:0x207:0x0], intent: layout flags 0 00010000:00010000:2.0:1769074847.669415:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 66757 initiator: MDT0 00010000:00010000:2.0:1769074847.669425:0:66757:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66757 initiator: MDT0 00010000:00010000:2.0:1769074847.669434:0:66757:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 66757 initiator: MDT0 00000100:00100000:2.0:1769074847.669455:0:66757:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617028224:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074847.669542:0:66757:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5ee40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074847.672799:0:66757:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617028224:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074847.672834:0:66757:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.672880:0:66757:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000002:00100000:2.0:1769074847.672895:0:66757:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9b03482c4380 x1855007617028224/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 376/896 e 0 to 0 dl 1769074902 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074847.672905:0:66757:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000002:00010000:2.0:1769074847.672918:0:66757:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.672932:0:66757:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 16 res 8589935621/519 (0 0) 00000080:00200000:2.0:1769074847.672938:0:66757:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200000405:0x207:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1769074847.672954:0:66757:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c832 00000080:00010000:2.0:1769074847.672962:0:66757:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c832 from it ffffb43641c77bc8 00010000:00010000:2.0:1769074847.672967:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.672976:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.673010:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.673022:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074847.673030:0:66757:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000405:0x207:0x0](0000000050727208) being reconfigured ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.673075:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.673082:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074847.673097:0:66757:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074847.673109:0:66757:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x207:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074847.673112:0:66757:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074847.673119:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074847.673130:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074847.673137:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074847.673158:0:66757:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1769074847.673170:0:66757:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.673195:0:66757:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074847.673203:0:66757:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000405:0x207:0x0](0000000050727208) being reconfigured ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074847.673220:0:66757:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 1/0, layout version 7 restore needed 0 00020000:00200000:2.0:1769074847.673225:0:66757:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x207:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074847.673228:0:66757:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074847.673231:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074847.673505:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074847.673511:0:66757:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074847.673521:0:66757:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1769074847.673534:0:66757:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 1048576 blocks 2048 [cma]time: 1769074846 1769074846 0 by oinfo size 1048576 blocks 2048 [cma]time 1769074847 1769074846 1769074846 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074847.673559:0:66757:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 2097152 blocks 2048 [cma]time: 1769074846 1769074846 0 by oinfo size 2097152 blocks 2048 [cma]time 1769074847 1769074846 1769074846 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074847.673574:0:66757:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 2097152 blocks 4096 [cma]time: 1769074847 1769074847 0 by oinfo size 2097152 blocks 4096 [cma]time 1769074847 1769074847 0 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c816 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55726 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074847.673587:0:66757:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 0 blocks 0 [cma]time 1769074847 1769074847 0 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1400/0xb7d6bf619879c81d lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x2ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd5572d expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074847.673704:0:66757:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x207:0x0]: layout version change: 4294967294 -> 8 00010000:00010000:2.0:1769074847.673709:0:66757:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074847.673716:0:66757:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200008:2.0:1769074847.673753:0:66757:0:(file.c:424:ll_file_release()) START file f204b.sanity-flr:[0x200000405:0x207:0x0](ffff9b0355d41988), flags 154002 00000002:00100000:2.0:1769074847.673794:0:66757:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0365a38700 x1855007617023360/t8589944109(8589944109) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074902 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074847.673804:0:66757:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074847.673810:0:66757:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c4380 x1855007617028480/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074847.673831:0:66757:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617028480:192.168.204.160@tcp:35:lfs.0 00000100:00100000:2.0:1769074847.673981:0:66757:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5ee40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074847.680933:0:66757:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66757:1855007617028480:192.168.204.160@tcp:35:lfs.0 00000002:00100000:2.0:1769074847.680997:0:66757:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0365a38700 x1855007617023360/t8589944109(8589944109) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074902 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:0.0:1769074847.736316:0:66760:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074847.736326:0:66760:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074847.736352:0:66760:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074847.736370:0:66760:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.736385:0:66760:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.736393:0:66760:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074847.736400:0:66760:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074847.736404:0:66760:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:0.0:1769074847.736411:0:66760:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641c97cb8 00010000:00010000:0.0:1769074847.736415:0:66760:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.736423:0:66760:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.736539:0:66760:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074847.736549:0:66760:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074847.736556:0:66760:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.736565:0:66760:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.736571:0:66760:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074847.736575:0:66760:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074847.736578:0:66760:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:0.0:1769074847.736581:0:66760:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641c97bb0 00010000:00010000:0.0:1769074847.736585:0:66760:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.736592:0:66760:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.736600:0:66760:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074847.736606:0:66760:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:0.0:1769074847.736623:0:66760:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffec734b9ac 00000080:00200008:0.0:1769074847.736635:0:66760:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:1.0:1769074847.739071:0:66762:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074847.739099:0:66762:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074847.739118:0:66762:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.739138:0:66762:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.739146:0:66762:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074847.739156:0:66762:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074847.739160:0:66762:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074847.739167:0:66762:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641dd3bf8 00010000:00010000:1.0:1769074847.739171:0:66762:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.739179:0:66762:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.739205:0:66762:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074847.739213:0:66762:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:1.0:1769074847.739219:0:66762:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074847.739231:0:66762:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), file ffff9b0349282f00, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1769074847.739255:0:66762:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), intent=open 00000002:00010000:1.0:1769074847.739270:0:66762:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x8:0x0], intent: open flags 0304001 00010000:00010000:1.0:1769074847.739325:0:66762:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66762 initiator: MDT0 00010000:00010000:1.0:1769074847.739333:0:66762:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66762 initiator: MDT0 00000020:00100000:1.0:1769074847.739344:0:66762:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074847.739357:0:66762:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ce9180 x1855007617029120/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074847.739372:0:66762:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66762 initiator: MDT0 00000100:00100000:1.0:1769074847.739394:0:66762:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66762:1855007617029120:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074847.739728:0:66762:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210480 going to sleep for 50 seconds 00000100:00100000:2.0:1769074847.744983:0:66762:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66762:1855007617029120:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074847.745023:0:66762:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66762 initiator: MDT0 00010000:00010000:2.0:1769074847.745036:0:66762:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 66762 initiator: MDT0 00010000:00010000:2.0:1769074847.745044:0:66762:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 66762 initiator: MDT0 00010000:00010000:2.0:1769074847.745050:0:66762:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 66762 initiator: MDT0 00010000:00010000:2.0:1769074847.745057:0:66762:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 66762 initiator: MDT0 00010000:00010000:2.0:1769074847.745063:0:66762:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035005a800 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074847.745067:0:66762:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 66762 initiator: MDT0 00010000:00010000:2.0:1769074847.745074:0:66762:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 66762 initiator: MDT0 00010000:00010000:2.0:1769074847.745081:0:66762:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 66762 initiator: MDT0 00010000:00010000:2.0:1769074847.745088:0:66762:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074847.745093:0:66762:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 66762 initiator: MDT0 00010000:00010000:2.0:1769074847.745099:0:66762:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c840 lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 66762 initiator: MDT0 00000002:00100000:2.0:1769074847.745124:0:66762:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0342ce9180 x1855007617029120/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074902 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074847.745137:0:66762:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x8:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074847.745418:0:66762:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074847.745440:0:66762:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074847.745454:0:66762:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.745466:0:66762:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.745473:0:66762:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074847.745478:0:66762:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074847.745482:0:66762:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074847.745487:0:66762:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641dd3bf8 00010000:00010000:2.0:1769074847.745490:0:66762:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.745499:0:66762:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.745511:0:66762:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074847.745518:0:66762:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:2.0:1769074847.745525:0:66762:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 44 00000080:00200008:2.0:1769074847.745534:0:66762:0:(file.c:1001:ll_file_open()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00000080:00200080:2.0:1769074847.745585:0:66762:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8) cmd=c1406916 arg=1809a10 00000100:00100000:2.0:1769074847.746913:0:66762:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66762:1855007617029248:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074847.747108:0:66762:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5ee40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074847.761476:0:66762:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66762:1855007617029248:192.168.204.160@tcp:34:lfs.0 00000080:00200008:1.0:1769074847.766944:0:66762:0:(file.c:424:ll_file_release()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00010000:00010000:1.0:1769074847.766983:0:66762:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074847.791123:0:66767:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074847.791136:0:66767:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074847.791158:0:66767:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074847.791175:0:66767:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.791881:0:66767:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.791893:0:66767:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074847.791901:0:66767:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074847.791904:0:66767:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074847.791913:0:66767:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641d47cb8 00010000:00010000:1.0:1769074847.791916:0:66767:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.792897:0:66767:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.793171:0:66767:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074847.793388:0:66767:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074847.793403:0:66767:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.793415:0:66767:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.793422:0:66767:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074847.793428:0:66767:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074847.793431:0:66767:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074847.793437:0:66767:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641d47bb0 00010000:00010000:1.0:1769074847.793440:0:66767:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.793447:0:66767:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.793455:0:66767:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074847.793464:0:66767:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074847.793485:0:66767:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7fff0dde70ec 00000080:00200008:1.0:1769074847.793498:0:66767:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:0.0:1769074847.794988:0:66770:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074847.795019:0:66770:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074847.795033:0:66770:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.795045:0:66770:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.795052:0:66770:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074847.795058:0:66770:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074847.795062:0:66770:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:0.0:1769074847.795069:0:66770:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641dd3bf8 00010000:00010000:0.0:1769074847.795072:0:66770:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.795079:0:66770:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.795092:0:66770:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074847.795098:0:66770:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:0.0:1769074847.795104:0:66770:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 1 00000080:00200000:0.0:1769074847.795115:0:66770:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), file ffff9b0360858a80, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1769074847.795121:0:66770:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), intent=open 00000002:00010000:0.0:1769074847.795137:0:66770:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x8:0x0], intent: open flags 0304001 00010000:00010000:0.0:1769074847.795196:0:66770:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66770 initiator: MDT0 00010000:00010000:0.0:1769074847.795203:0:66770:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66770 initiator: MDT0 00000020:00100000:0.0:1769074847.795216:0:66770:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074847.795229:0:66770:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a38380 x1855007617029504/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074847.795273:0:66770:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66770 initiator: MDT0 00000100:00100000:0.0:1769074847.795317:0:66770:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66770:1855007617029504:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074847.795856:0:66770:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1aa80 going to sleep for 50 seconds 00000100:00100000:0.0:1769074847.800203:0:66770:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66770:1855007617029504:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074847.800228:0:66770:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66770 initiator: MDT0 00010000:00010000:0.0:1769074847.800276:0:66770:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 66770 initiator: MDT0 00010000:00010000:0.0:1769074847.800283:0:66770:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 66770 initiator: MDT0 00010000:00010000:0.0:1769074847.800289:0:66770:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 66770 initiator: MDT0 00010000:00010000:0.0:1769074847.800296:0:66770:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 66770 initiator: MDT0 00010000:00010000:0.0:1769074847.800302:0:66770:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0348215200 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074847.800306:0:66770:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 66770 initiator: MDT0 00010000:00010000:0.0:1769074847.800313:0:66770:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 66770 initiator: MDT0 00010000:00010000:0.0:1769074847.800325:0:66770:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 66770 initiator: MDT0 00010000:00010000:0.0:1769074847.800332:0:66770:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074847.800338:0:66770:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 66770 initiator: MDT0 00010000:00010000:0.0:1769074847.800354:0:66770:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c847 lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 66770 initiator: MDT0 00000002:00100000:0.0:1769074847.800381:0:66770:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0365a38380 x1855007617029504/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074902 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074847.800396:0:66770:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x8:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1769074847.800623:0:66770:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074847.800646:0:66770:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074847.800671:0:66770:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.800682:0:66770:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.800689:0:66770:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074847.800694:0:66770:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074847.800697:0:66770:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:0.0:1769074847.800702:0:66770:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641dd3bf8 00010000:00010000:0.0:1769074847.800705:0:66770:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074847.800712:0:66770:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074847.800725:0:66770:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074847.800733:0:66770:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:0.0:1769074847.800739:0:66770:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 44 00000080:00200008:0.0:1769074847.800747:0:66770:0:(file.c:1001:ll_file_open()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00000080:00200080:0.0:1769074847.800791:0:66770:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8) cmd=c1406916 arg=ef1a10 00000100:00100000:0.0:1769074847.800859:0:66770:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66770:1855007617029632:192.168.204.160@tcp:34:lfs.0 00000100:00100000:0.0:1769074847.800939:0:66770:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1aa80 going to sleep for 50 seconds 00000100:00100000:0.0:1769074847.806673:0:66770:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66770:1855007617029632:192.168.204.160@tcp:34:lfs.0 00000080:00200008:0.0:1769074847.806833:0:66770:0:(file.c:424:ll_file_release()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00010000:00010000:0.0:1769074847.806876:0:66770:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074847.835743:0:66774:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074847.835753:0:66774:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074847.835775:0:66774:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074847.835793:0:66774:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.835814:0:66774:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.835823:0:66774:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074847.835830:0:66774:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074847.835845:0:66774:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074847.835853:0:66774:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641c97cb8 00010000:00010000:1.0:1769074847.835857:0:66774:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.835865:0:66774:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.835962:0:66774:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074847.835971:0:66774:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074847.835977:0:66774:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.835986:0:66774:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.835991:0:66774:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074847.835995:0:66774:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074847.835998:0:66774:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074847.836002:0:66774:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641c97bb0 00010000:00010000:1.0:1769074847.836005:0:66774:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074847.836016:0:66774:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074847.836025:0:66774:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074847.836032:0:66774:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074847.836046:0:66774:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffd15a5981c 00000080:00200008:1.0:1769074847.836058:0:66774:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:2.0:1769074847.840518:0:66777:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074847.840553:0:66777:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074847.840570:0:66777:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.840590:0:66777:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.840598:0:66777:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074847.840606:0:66777:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074847.840610:0:66777:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074847.840617:0:66777:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641d4fbf8 00010000:00010000:2.0:1769074847.840621:0:66777:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.840629:0:66777:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.840643:0:66777:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074847.840651:0:66777:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:2.0:1769074847.840668:0:66777:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 1 00000080:00200000:2.0:1769074847.840679:0:66777:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), file ffff9b0351f71380, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074847.840686:0:66777:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), intent=open 00000002:00010000:2.0:1769074847.840701:0:66777:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x8:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074847.840758:0:66777:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c84e lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66777 initiator: MDT0 00010000:00010000:2.0:1769074847.840765:0:66777:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c84e lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66777 initiator: MDT0 00000020:00100000:2.0:1769074847.840778:0:66777:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074847.840790:0:66777:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c4380 x1855007617029888/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074847.840803:0:66777:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c84e lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66777 initiator: MDT0 00000100:00100000:2.0:1769074847.840826:0:66777:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66777:1855007617029888:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074847.841149:0:66777:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5ee40 going to sleep for 50 seconds 00000080:00200008:0.0:1769074847.877512:0:66777:0:(file.c:424:ll_file_release()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00010000:00010000:0.0:1769074847.877554:0:66777:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074847.899506:0:66782:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074847.899518:0:66782:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074847.899543:0:66782:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074847.899562:0:66782:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.899578:0:66782:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.899585:0:66782:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074847.899592:0:66782:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074847.899595:0:66782:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074847.899602:0:66782:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641dd3cb8 00010000:00010000:2.0:1769074847.899606:0:66782:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.899614:0:66782:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.899806:0:66782:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074847.899815:0:66782:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074847.899822:0:66782:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.899846:0:66782:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.899852:0:66782:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074847.899856:0:66782:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074847.899859:0:66782:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074847.899863:0:66782:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641dd3bb0 00010000:00010000:2.0:1769074847.899866:0:66782:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.899874:0:66782:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.899882:0:66782:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:2.0:1769074847.899890:0:66782:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:2.0:1769074847.899905:0:66782:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffcfe13133c 00000080:00200008:2.0:1769074847.899918:0:66782:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200008:2.0:1769074847.951802:0:66789:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074847.951813:0:66789:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074847.951847:0:66789:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074847.951865:0:66789:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.951879:0:66789:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.951888:0:66789:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074847.951895:0:66789:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074847.951899:0:66789:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074847.951907:0:66789:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641dd3cb8 00010000:00010000:2.0:1769074847.951911:0:66789:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.951919:0:66789:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.952060:0:66789:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074847.952069:0:66789:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074847.952076:0:66789:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.952084:0:66789:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.952091:0:66789:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074847.952095:0:66789:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074847.952098:0:66789:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074847.952102:0:66789:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641dd3bb0 00010000:00010000:2.0:1769074847.952105:0:66789:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074847.952112:0:66789:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074847.952119:0:66789:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:2.0:1769074847.952128:0:66789:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:2.0:1769074847.952143:0:66789:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffcf3c9bc4c 00000080:00200008:2.0:1769074847.952155:0:66789:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200008:1.0:1769074848.008963:0:66796:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074848.008973:0:66796:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074848.008996:0:66796:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074848.009119:0:66796:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074848.009135:0:66796:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074848.009143:0:66796:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074848.009150:0:66796:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074848.009154:0:66796:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074848.009160:0:66796:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641dc3cb8 00010000:00010000:1.0:1769074848.009164:0:66796:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074848.009172:0:66796:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074848.009554:0:66796:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074848.009564:0:66796:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074848.009572:0:66796:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074848.009581:0:66796:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074848.009588:0:66796:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074848.009592:0:66796:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074848.009594:0:66796:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074848.009599:0:66796:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641dc3bb0 00010000:00010000:1.0:1769074848.009601:0:66796:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074848.009609:0:66796:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074848.009618:0:66796:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074848.009626:0:66796:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074848.009644:0:66796:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffe9fc7905c 00000080:00200008:1.0:1769074848.009741:0:66796:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:1.0:1769074848.010868:0:66801:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074848.010886:0:66801:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074848.010896:0:66801:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074848.010908:0:66801:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074848.010914:0:66801:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074848.010920:0:66801:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074848.010923:0:66801:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:1.0:1769074848.010928:0:66801:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641e57bf8 00010000:00010000:1.0:1769074848.010932:0:66801:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074848.010939:0:66801:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074848.010950:0:66801:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074848.010956:0:66801:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:1.0:1769074848.010962:0:66801:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 1 00000080:00200000:1.0:1769074848.010972:0:66801:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), file ffff9b0353237980, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1769074848.010978:0:66801:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), intent=open 00000002:00010000:1.0:1769074848.010993:0:66801:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x8:0x0], intent: open flags 0304001 00010000:00010000:1.0:1769074848.011048:0:66801:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66801 initiator: MDT0 00010000:00010000:1.0:1769074848.011056:0:66801:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66801 initiator: MDT0 00000020:00100000:1.0:1769074848.011068:0:66801:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074848.011080:0:66801:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342cea300 x1855007617031040/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074848.011089:0:66801:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66801 initiator: MDT0 00000100:00100000:1.0:1769074848.011111:0:66801:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66801:1855007617031040:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074848.011455:0:66801:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0349210480 going to sleep for 50 seconds 00000100:00100000:2.0:1769074848.024515:0:66801:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66801:1855007617031040:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074848.024541:0:66801:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66801 initiator: MDT0 00010000:00010000:2.0:1769074848.030289:0:66801:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 66801 initiator: MDT0 00010000:00010000:2.0:1769074848.030314:0:66801:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 66801 initiator: MDT0 00010000:00010000:2.0:1769074848.030322:0:66801:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 66801 initiator: MDT0 00010000:00010000:2.0:1769074848.030330:0:66801:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 66801 initiator: MDT0 00010000:00010000:2.0:1769074848.030336:0:66801:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035005b200 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074848.030342:0:66801:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 66801 initiator: MDT0 00010000:00010000:2.0:1769074848.030349:0:66801:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 66801 initiator: MDT0 00010000:00010000:2.0:1769074848.030357:0:66801:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 66801 initiator: MDT0 00010000:00010000:2.0:1769074848.030365:0:66801:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074848.030372:0:66801:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 66801 initiator: MDT0 00010000:00010000:2.0:1769074848.030379:0:66801:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c863 lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 66801 initiator: MDT0 00000002:00100000:2.0:1769074848.030424:0:66801:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0342cea300 x1855007617031040/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074903 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074848.030438:0:66801:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x8:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074848.030737:0:66801:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074848.030760:0:66801:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074848.030773:0:66801:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074848.030784:0:66801:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074848.030792:0:66801:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074848.030797:0:66801:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074848.030801:0:66801:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074848.030806:0:66801:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641e57bf8 00010000:00010000:2.0:1769074848.030810:0:66801:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074848.030817:0:66801:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074848.030830:0:66801:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074848.030851:0:66801:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:2.0:1769074848.030858:0:66801:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 44 00000080:00200008:2.0:1769074848.030866:0:66801:0:(file.c:1001:ll_file_open()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00000080:00200080:2.0:1769074848.030914:0:66801:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8) cmd=c1406916 arg=11efa10 00000100:00100000:2.0:1769074848.030976:0:66801:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66801:1855007617031168:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074848.031616:0:66801:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5ec00 going to sleep for 50 seconds 00000100:00100000:1.0:1769074848.042305:0:66801:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66801:1855007617031168:192.168.204.160@tcp:34:lfs.0 00000080:00200008:1.0:1769074848.042466:0:66801:0:(file.c:424:ll_file_release()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00010000:00010000:1.0:1769074848.042498:0:66801:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074848.060475:0:66802:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074848.060508:0:66802:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074848.060526:0:66802:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074848.060541:0:66802:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074848.060548:0:66802:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074848.060556:0:66802:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074848.060559:0:66802:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074848.060566:0:66802:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641c97bf8 00010000:00010000:2.0:1769074848.060570:0:66802:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074848.060579:0:66802:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074848.060597:0:66802:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074848.060604:0:66802:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=27 00000080:00200000:2.0:1769074848.060609:0:66802:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 1 00000080:00200000:2.0:1769074848.060620:0:66802:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), file ffff9b0344c45500, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074848.060626:0:66802:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8), intent=open 00000002:00010000:2.0:1769074848.060640:0:66802:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204b.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x8:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074848.060697:0:66802:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 66802 initiator: MDT0 00010000:00010000:2.0:1769074848.060705:0:66802:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66802 initiator: MDT0 00000020:00100000:2.0:1769074848.060716:0:66802:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074848.060730:0:66802:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c5180 x1855007617031424/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074848.060740:0:66802:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66802 initiator: MDT0 00000100:00100000:2.0:1769074848.060760:0:66802:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66802:1855007617031424:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074848.060957:0:66802:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5e900 going to sleep for 50 seconds 00000100:00100000:2.0:1769074848.064895:0:66802:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66802:1855007617031424:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074848.064922:0:66802:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 66802 initiator: MDT0 00010000:00010000:2.0:1769074848.064935:0:66802:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 66802 initiator: MDT0 00010000:00010000:2.0:1769074848.064942:0:66802:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 66802 initiator: MDT0 00010000:00010000:2.0:1769074848.064950:0:66802:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 66802 initiator: MDT0 00010000:00010000:2.0:1769074848.064957:0:66802:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 66802 initiator: MDT0 00010000:00010000:2.0:1769074848.064963:0:66802:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365861e00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074848.064968:0:66802:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 66802 initiator: MDT0 00010000:00010000:2.0:1769074848.064975:0:66802:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 66802 initiator: MDT0 00010000:00010000:2.0:1769074848.064985:0:66802:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 66802 initiator: MDT0 00010000:00010000:2.0:1769074848.064993:0:66802:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074848.065000:0:66802:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 66802 initiator: MDT0 00010000:00010000:2.0:1769074848.065006:0:66802:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c86a lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 66802 initiator: MDT0 00000002:00100000:2.0:1769074848.065032:0:66802:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b03482c5180 x1855007617031424/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074903 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074848.065045:0:66802:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x8:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074848.065185:0:66802:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074848.065207:0:66802:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074848.065221:0:66802:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074848.065966:0:66802:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074848.065980:0:66802:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074848.065989:0:66802:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074848.065993:0:66802:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074848.066000:0:66802:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641c97bf8 00010000:00010000:2.0:1769074848.066003:0:66802:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074848.066011:0:66802:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074848.066028:0:66802:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074848.066037:0:66802:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=267 00000080:00200000:2.0:1769074848.066045:0:66802:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 44 00000080:00200008:2.0:1769074848.066055:0:66802:0:(file.c:1001:ll_file_open()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00000080:00200080:2.0:1769074848.066101:0:66802:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8) cmd=c0086915 arg=c5e3d0 00000100:00100000:2.0:1769074848.066166:0:66802:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66802:1855007617031552:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074848.066506:0:66802:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5e900 going to sleep for 50 seconds 00000100:00100000:2.0:1769074848.070660:0:66802:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:66802:1855007617031552:192.168.204.160@tcp:34:lfs.0 00000080:00200008:2.0:1769074848.070848:0:66802:0:(file.c:424:ll_file_release()) START file d204b.sanity-flr:[0x200000405:0x206:0x0](ffff9b03606fa1c8), flags 304000 00010000:00010000:2.0:1769074848.070882:0:66802:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164d000/0xb7d6bf619879c7bb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x206:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd555dd expref: -99 pid: 66732 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074848.148640:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074849.174578:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074849.363463:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617031808:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074849.363578:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617032064:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074849.363615:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1769074849.363882:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617031680:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074849.363919:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617031936:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074849.363942:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617032192:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074849.372434:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617031680:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074849.372464:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617031936:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074849.372483:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617032192:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074849.372503:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617031808:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074849.372532:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617032064:192.168.204.160@tcp:400:kworker.0 00010000:00010000:2.0:1769074850.195642:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074851.275795:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074853.106592:0:67226:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b035013cc00 going to sleep for 50 seconds 00000100:00100000:2.0:1769074853.120488:0:67226:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67226:1855007617032320:192.168.204.160@tcp:101:rm.0 00010000:00010000:2.0:1769074853.120517:0:67226:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x8:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.120545:0:67226:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.120553:0:67226:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.120565:0:67226:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.120582:0:67226:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.120596:0:67226:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.120604:0:67226:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.120612:0:67226:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074853.120625:0:67226:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b0352a1a680 x1855007617032320/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/1136 e 0 to 0 dl 1769074908 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074853.120637:0:67226:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1769074853.120648:0:67226:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074853.120655:0:67226:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/519 (0 0) 00000080:00200000:2.0:1769074853.120660:0:67226:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x8:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074853.120706:0:67226:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x207:0x0], updating i_size 2097152 00000080:00200000:2.0:1769074853.120708:0:67226:0:(namei.c:151:ll_iget()) got inode: [0x200000405:0x207:0x0](ffff9b0355d41988): rc = 0 00000080:00200000:2.0:1769074853.120724:0:67226:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x207:0x0]: layout version change: 8 -> 8 00000080:00010000:2.0:1769074853.120729:0:67226:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c871 00000080:00010000:2.0:1769074853.120740:0:67226:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) 00000080:00010000:2.0:1769074853.120744:0:67226:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c871 from it ffffb43641c17b20 00010000:00010000:2.0:1769074853.120757:0:67226:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074853.120765:0:67226:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 3 00000080:00200008:2.0:1769074853.120804:0:67226:0:(file.c:6252:ll_getattr_dentry()) START file f204b.sanity-flr:[0x200000405:0x207:0x0](ffff9b0355d41988), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074853.120812:0:67226:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988),name=f204b.sanity-flr 00000002:00010000:2.0:1769074853.120835:0:67226:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x207:0x0]) in obj [0x200000bd0:0x8:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074853.120843:0:67226:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074853.120853:0:67226:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074853.120859:0:67226:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x8:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074853.120864:0:67226:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) 00000080:00010000:2.0:1769074853.120867:0:67226:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x207:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c871 00000080:00010000:2.0:1769074853.120871:0:67226:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c871 from it ffffb43641c17cb8 00010000:00010000:2.0:1769074853.120874:0:67226:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074853.120881:0:67226:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074853.120945:0:67226:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074853.120954:0:67226:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074853.120962:0:67226:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.120970:0:67226:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074853.120976:0:67226:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074853.120980:0:67226:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074853.120983:0:67226:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074853.120986:0:67226:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641c17c10 00010000:00010000:2.0:1769074853.120989:0:67226:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.120997:0:67226:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074853.121004:0:67226:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074853.121009:0:67226:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204b.sanity-flr, flags=24 00000080:00200000:2.0:1769074853.121013:0:67226:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 1 00000080:00200000:2.0:1769074853.121017:0:67226:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204b.sanity-flr, flags=0 00000080:00200000:2.0:1769074853.121019:0:67226:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x206:0x0](ffff9b03606fa1c8), inode mode 41ed mask 3 00000080:00200000:2.0:1769074853.121023:0:67226:0:(namei.c:2354:ll_unlink()) VFS Op:name=f204b.sanity-flr, dir=[0x200000405:0x206:0x0](ffff9b03606fa1c8) 00010000:00010000:2.0:1769074853.121042:0:67226:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074853.121051:0:67226:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x207:0x0]: losing layout lock 00010000:00010000:2.0:1769074853.121058:0:67226:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074853.121063:0:67226:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x207:0x0]: losing layout lock 00010000:00010000:2.0:1769074853.121067:0:67226:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c79f lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x207:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55574 expref: -99 pid: 66727 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.121098:0:67226:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 1/0,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074853.121102:0:67226:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074853.121106:0:67226:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c79f lrc: 1/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55574 expref: -99 pid: 66727 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074853.121109:0:67226:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1769074853.121111:0:67226:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c832 lrc: 0/0,0 mode: --/CR res: [0x200000405:0x207:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55750 expref: -99 pid: 66757 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074853.121120:0:67226:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd0800/0xb7d6bf619879c871 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd558bc expref: -99 pid: 67226 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074853.121124:0:67226:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342cd1600/0xb7d6bf619879c79f lrc: 0/0,0 mode: --/PR res: [0x200000405:0x207:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55574 expref: -99 pid: 66727 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074853.121137:0:67226:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074853.121142:0:67226:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c5180 x1855007617032576/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 512/2128 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074853.121158:0:67226:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67226:1855007617032576:192.168.204.160@tcp:36:rm.0 00000100:00100000:2.0:1769074853.121224:0:67226:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0358a32e40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074853.125542:0:67226:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67226:1855007617032576:192.168.204.160@tcp:36:rm.0 00000080:00200000:2.0:1769074853.125615:0:67226:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x207:0x0] ignore/verify layout 1/0, layout version 7 restore needed 0 00020000:00200000:2.0:1769074853.125627:0:67226:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x207:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074853.125630:0:67226:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074853.125635:0:67226:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074853.125644:0:67226:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074853.125648:0:67226:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074853.125666:0:67226:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x207:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074853.125673:0:67226:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200000405:0x207:0x0](ffff9b0355d41988) 00000100:00100000:0.0:1769074853.151042:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074853.154808:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074853.155224:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625238144 00000100:00100000:1.0:1769074853.155298:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625238144 00000100:00100000:1.0:1769074853.155351:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625238272 00000100:00100000:1.0:1769074853.155354:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625238272 00000100:00100000:1.0:1769074853.155372:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8945 00000100:00100000:1.0:1769074853.155381:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a38380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:27843:x1855007625238144:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074853.155396:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074853.155509:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074853.155516:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a38380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:27843:x1855007625238144:12345-192.168.204.160@tcp:104: Request processed in 137us (4489us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074853.155523:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8945 00000100:00100000:1.0:1769074853.155541:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8946 00000100:00100000:1.0:1769074853.155547:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a3a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:28955:x1855007625238272:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074853.155555:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c816 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55726 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.155877:0:39339:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074853.155885:0:39339:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.155894:0:39339:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0342cd0200 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:2.0:1769074853.155908:0:39339:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x428400030000 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.155952:0:39339:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.155959:0:39339:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074853.155985:0:39339:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.155994:0:39339:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd0200/0xb7d6bf619879c824 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd5573b expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.156013:0:39339:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074853.156016:0:39339:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c816 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55726 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.156023:0:39339:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0342cd1200 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:2.0:1769074853.156027:0:39339:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c816 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x428400010000 nid: local remote: 0x7fe2323adfd55726 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.156046:0:39339:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c816 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55726 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.156052:0:39339:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074853.156058:0:39339:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c816 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55726 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.156064:0:39339:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1200/0xb7d6bf619879c816 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55726 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074853.156076:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074853.156078:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074853.156079:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074853.157258:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:1.0:1769074853.157272:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a3a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:28955:x1855007625238272:12345-192.168.204.160@tcp:104: Request processed in 1721us (2472us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074853.157282:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8946 00010000:00010000:2.0:1769074853.159021:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074853.188775:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617032832:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074853.337316:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074854.878149:0:67367:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204c: FLR write/stale/resync test with component removal ========================================================== 04:40:53 (1769074853) 00000080:00200008:2.0:1769074855.272811:0:67385:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074855.272822:0:67385:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074855.272846:0:67385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074855.272864:0:67385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.272880:0:67385:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.272888:0:67385:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074855.272895:0:67385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074855.272899:0:67385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074855.272906:0:67385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641d4fcb8 00010000:00010000:2.0:1769074855.272910:0:67385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.272919:0:67385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.272994:0:67385:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074855.273002:0:67385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074855.273008:0:67385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.273016:0:67385:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.273023:0:67385:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074855.273027:0:67385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074855.273029:0:67385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074855.273032:0:67385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641d4fbb0 00010000:00010000:2.0:1769074855.273035:0:67385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.273042:0:67385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.273051:0:67385:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:2.0:1769074855.273058:0:67385:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:2.0:1769074855.273075:0:67385:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffd1ef62b4c 00000080:00200008:2.0:1769074855.273090:0:67385:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:2.0:1769074855.273111:0:67385:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074855.273118:0:67385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074855.273125:0:67385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.273133:0:67385:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.273138:0:67385:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074855.273142:0:67385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074855.273144:0:67385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074855.273148:0:67385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641d4fbb0 00010000:00010000:2.0:1769074855.273151:0:67385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.273158:0:67385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.273166:0:67385:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:2.0:1769074855.273170:0:67385:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 100000 00000080:00200080:2.0:1769074855.273179:0:67385:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=8080667f arg=7ffd1ef61820 00000080:00200008:2.0:1769074855.273551:0:67385:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 100000 00000080:00200008:2.0:1769074855.273871:0:67385:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074855.273876:0:67385:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074855.273887:0:67385:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074855.273895:0:67385:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.273910:0:67385:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.273917:0:67385:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074855.273922:0:67385:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074855.273926:0:67385:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:2.0:1769074855.273930:0:67385:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641d4fcb8 00010000:00010000:2.0:1769074855.273933:0:67385:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.273941:0:67385:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.344354:0:67391:0:(llite_lib.c:2766:ll_statfs()) VFS Op:sb=lustre (ffff9b0342ad2800) / proj=0 00000080:00000004:2.0:1769074855.344366:0:67391:0:(obd_class.h:1083:obd_statfs()) lustre-clilmv-ffff9b0342ad2800: age 1840, max_age 1848 00800000:00000004:2.0:1769074855.344380:0:67391:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff9b0342ad2800: age 1840, max_age 1848 00000100:00100000:2.0:1769074855.344426:0:67391:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job stat:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67391:1855007617033728:192.168.204.160@tcp:41:stat.0 00000100:00100000:2.0:1769074855.344516:0:67391:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074855.350827:0:67391:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job stat:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67391:1855007617033728:192.168.204.160@tcp:41:stat.0 00800000:00000004:2.0:1769074855.350859:0:67391:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff9b0342ad2800: update ffff9b03512de6d0 cache blocks 1797775/1916558 objects 523379/524056 00000080:00000004:2.0:1769074855.350866:0:67391:0:(obd_class.h:1109:obd_statfs()) lustre-clilmv-ffff9b0342ad2800: update ffff9b03512db520 cache blocks 1797775/1916558 objects 523379/524056 00000080:00000004:2.0:1769074855.350871:0:67391:0:(llite_lib.c:2629:ll_statfs_internal()) MDC blocks 1797775/1916558 objects 523379/524056 00010000:00010000:1.0:1769074855.380656:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1769074855.429629:0:67395:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074855.429662:0:67395:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074855.429681:0:67395:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074855.429698:0:67395:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074855.429717:0:67395:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074855.429725:0:67395:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074855.429729:0:67395:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c759 00000080:00010000:0.0:1769074855.429736:0:67395:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c759 from it ffffb43641c87b58 00010000:00010000:0.0:1769074855.429740:0:67395:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074855.429749:0:67395:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b034164c200/0xb7d6bf619879c759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5549b expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074855.429765:0:67395:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074855.429779:0:67395:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d204c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), flags=13 00000080:00200000:0.0:1769074855.429783:0:67395:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d204c.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=getattr 00000002:00010000:0.0:1769074855.429796:0:67395:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074855.429840:0:67395:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348214e00/0xb7d6bf619879c878 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67395 initiator: MDT0 00010000:00010000:0.0:1769074855.429848:0:67395:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348214e00/0xb7d6bf619879c878 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67395 initiator: MDT0 00010000:00010000:0.0:1769074855.429858:0:67395:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348214e00/0xb7d6bf619879c878 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67395 initiator: MDT0 00000100:00100000:0.0:1769074855.429881:0:67395:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67395:1855007617033856:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074855.430262:0:67395:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a9c0 going to sleep for 50 seconds 00000080:00200000:0.0:1769074855.505186:0:67398:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074855.505217:0:67398:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074855.505938:0:67398:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67398 initiator: MDT0 00010000:00010000:0.0:1769074855.505952:0:67398:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67398 initiator: MDT0 00010000:00010000:0.0:1769074855.506461:0:67398:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67398 initiator: MDT0 00000100:00100000:0.0:1769074855.506488:0:67398:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67398:1855007617034240:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074855.506794:0:67398:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ab40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074855.527906:0:67398:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b5cc00 going to sleep for 50 seconds 00000100:00100000:2.0:1769074855.537920:0:67398:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67398:1855007617034624:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074855.537948:0:67398:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 4/1,0 mode: --/CR res: [0x240000405:0x9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.537973:0:67398:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 4/1,0 mode: --/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.537985:0:67398:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 4/1,0 mode: PR/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.537997:0:67398:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 4/1,0 mode: PR/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.538008:0:67398:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 5/2,0 mode: PR/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.538015:0:67398:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 5/2,0 mode: PR/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.538021:0:67398:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 4/1,0 mode: PR/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074855.538032:0:67398:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b0352a1a300 x1855007617034624/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 576/640 e 0 to 0 dl 1769074910 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074855.538046:0:67398:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 3/1,0 mode: PR/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.538059:0:67398:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 4 res 9663677445/9 (0 0) 00010000:00010000:2.0:1769074855.538068:0:67398:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 3/1,0 mode: PR/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.538075:0:67398:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 2/0,0 mode: PR/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.538111:0:67398:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x209:0x0], updating i_size 8192 00000080:00200000:2.0:1769074855.538115:0:67398:0:(namei.c:151:ll_iget()) got inode: [0x200000405:0x209:0x0](ffff9b0355ce0908): rc = 0 00000080:00010000:2.0:1769074855.538124:0:67398:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x209:0x0](ffff9b0355ce0908) for lock 0xb7d6bf619879c886 00000080:00010000:2.0:1769074855.538136:0:67398:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x209:0x0](ffff9b0355ce0908) 00000080:00010000:2.0:1769074855.538141:0:67398:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c886 from it ffffb43641473a28 00010000:00010000:2.0:1769074855.538145:0:67398:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a600/0xb7d6bf619879c886 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x209:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5594f expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.538154:0:67398:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a600/0xb7d6bf619879c886 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x209:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5594f expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.538175:0:67398:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:2.0:1769074855.538197:0:67398:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), flags=13 00000080:00200000:2.0:1769074855.538201:0:67398:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=getattr 00000002:00010000:2.0:1769074855.538223:0:67398:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x9:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074855.538309:0:67398:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.538316:0:67398:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.538326:0:67398:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67398 initiator: MDT0 00000100:00100000:2.0:1769074855.538347:0:67398:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67398:1855007617034752:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074855.538427:0:67398:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55cc0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074855.542593:0:67398:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67398:1855007617034752:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074855.542698:0:67398:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.542722:0:67398:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.542728:0:67398:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.542734:0:67398:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.542740:0:67398:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.542745:0:67398:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365861000 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074855.542749:0:67398:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.542755:0:67398:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.542761:0:67398:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.542768:0:67398:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074855.542774:0:67398:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.542780:0:67398:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c89b lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67398 initiator: MDT0 00000002:00100000:2.0:1769074855.542800:0:67398:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9b03482c7480 x1855007617034752/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074910 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074855.542815:0:67398:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1769074855.542849:0:67398:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b800/0xb7d6bf619879c88d lrc: 2/1,0 mode: PR/PR res: [0x200000bd0:0x9:0x0].0x0 bits 0x12/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5595d expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.542862:0:67398:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b800/0xb7d6bf619879c88d lrc: 2/1,0 mode: PR/PR res: [0x200000bd0:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5595d expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00010000:2.0:1769074855.542871:0:67398:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c88d from it ffffb43641473940 00010000:00010000:2.0:1769074855.542875:0:67398:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b800/0xb7d6bf619879c88d lrc: 3/1,0 mode: PR/PR res: [0x200000bd0:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5595d expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.542883:0:67398:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b800/0xb7d6bf619879c88d lrc: 2/0,0 mode: PR/PR res: [0x200000bd0:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5595d expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.543057:0:67398:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074855.543074:0:67398:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074855.543145:0:67398:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.543154:0:67398:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.543161:0:67398:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074855.543167:0:67398:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074855.543171:0:67398:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074855.543176:0:67398:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641473be8 00010000:00010000:2.0:1769074855.543179:0:67398:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.543186:0:67398:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.543199:0:67398:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074855.543206:0:67398:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=16408 00000080:00200000:2.0:1769074855.543211:0:67398:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:2.0:1769074855.543214:0:67398:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204c.sanity-flr, flags=16392 00000080:00200000:2.0:1769074855.543331:0:67398:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074855.543340:0:67398:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074855.543348:0:67398:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.543356:0:67398:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.543362:0:67398:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074855.543366:0:67398:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074855.543368:0:67398:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074855.543371:0:67398:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641473b58 00010000:00010000:2.0:1769074855.543374:0:67398:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074855.543381:0:67398:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074855.543389:0:67398:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074855.543393:0:67398:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=13 00000080:00200008:2.0:1769074855.543399:0:67398:0:(file.c:6252:ll_getattr_dentry()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074855.543404:0:67398:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908),name=d204c.sanity-flr 00000002:00010000:2.0:1769074855.543416:0:67398:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x209:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074855.543423:0:67398:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 30 res 8589935621/521 (0 0) 00010000:00010000:2.0:1769074855.543460:0:67398:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c8a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.543467:0:67398:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c8a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67398 initiator: MDT0 00010000:00010000:2.0:1769074855.543476:0:67398:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861400/0xb7d6bf619879c8a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67398 initiator: MDT0 00000100:00100000:2.0:1769074855.543496:0:67398:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67398:1855007617034880:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074855.543576:0:67398:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55cc0 going to sleep for 50 seconds 00000100:00100000:0.0:1769074855.563335:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074855.563434:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625240832 00000100:00100000:1.0:1769074855.563445:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625240832 00000100:00100000:1.0:1769074855.563506:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8947 00000100:00100000:1.0:1769074855.563517:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a3aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6513:x1855007625240832:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074855.563543:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c8a9 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd559aa expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074855.563627:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074855.563634:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a3aa00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6513:x1855007625240832:12345-192.168.204.160@tcp:104: Request processed in 119us (309us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074855.563642:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8947 00010000:00010000:2.0:1769074857.431910:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074858.451620:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074858.550585:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074858.550630:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625241728 00000100:00100000:0.0:1769074858.550641:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625241728 00000100:00100000:0.0:1769074858.550707:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8948 00000100:00100000:0.0:1769074858.550718:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a38e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:9048:x1855007625241728:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074858.550734:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074858.550828:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074858.550844:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a38e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:9048:x1855007625241728:12345-192.168.204.160@tcp:104: Request processed in 128us (272us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074858.550852:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8948 00010000:00010000:2.0:1769074858.552417:0:57730:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074858.552431:0:57730:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074858.552443:0:57730:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0342cd1e00 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:2.0:1769074858.552449:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x428400030000 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074858.552499:0:57730:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 2/0,0 mode: --/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074858.552506:0:57730:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074858.552544:0:57730:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074858.552552:0:57730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1e00/0xb7d6bf619879c82b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->68157439) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55742 expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074858.552574:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074858.552575:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074858.552577:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074858.552586:0:2408:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074858.552605:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617036160:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074858.564724:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074858.565059:0:55997:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074858.565079:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625241856 00000100:00100000:0.0:1769074858.565091:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625241856 00000100:00100000:0.0:1769074858.565150:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8949 00000100:00100000:0.0:1769074858.565160:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0343955050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:27843:x1855007625241856:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074858.565176:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1400/0xb7d6bf619879c81d lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd5572d expref: -99 pid: 66757 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074858.565273:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074858.565281:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0343955050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:27843:x1855007625241856:12345-192.168.204.160@tcp:104: Request processed in 123us (567us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074858.565287:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8949 00000100:00100000:2.0:1769074858.566982:0:2409:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074858.567004:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617036288:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074858.569544:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617036160:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074858.577566:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617036288:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1769074859.477198:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074859.604402:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617036544:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074859.604561:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617036800:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074859.604606:0:2408:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:2.0:1769074859.604613:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2408:1855007617036416:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074859.604637:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617036672:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074859.604670:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617036928:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074859.608805:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617036800:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074859.620888:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2408:1855007617036416:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074859.620919:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617036672:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074859.620933:0:2408:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9b0342ad2800: committing for last_committed 17179874633 gen 2 00000100:00100000:2.0:1769074859.620981:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617036928:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074859.620994:0:2408:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944122 gen 1 00000100:00100000:2.0:1769074859.621060:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617036544:192.168.204.160@tcp:400:kworker.0 00010000:00010000:2.0:1769074862.550156:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074863.574772:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1769074863.700907:0:59411:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x204:0x0] ignore/verify layout 1/0, layout version 6 restore needed 0 00020000:00200000:1.0:1769074863.700929:0:59411:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074863.700983:0:59411:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x204:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074863.701001:0:59411:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x202:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074863.701008:0:59411:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x202:0x0]: flr state: 1, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074863.701012:0:59411:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1769074863.701014:0:59411:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1769074863.701021:0:59411:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x202:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1769074864.595567:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074864.724359:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617037184:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074864.724644:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617037440:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074864.758687:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617037184:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074864.758715:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617037440:192.168.204.160@tcp:400:kworker.0 00010000:00010000:1.0:1769074866.643581:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1769074867.675104:0:67421:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074867.675147:0:67421:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074867.675169:0:67421:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074867.675189:0:67421:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074867.675198:0:67421:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074867.675208:0:67421:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074867.675212:0:67421:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:0.0:1769074867.675221:0:67421:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb4364185bbf8 00010000:00010000:0.0:1769074867.679650:0:67421:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074867.679672:0:67421:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074867.679697:0:67421:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074867.679713:0:67421:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=25 00000080:00200000:0.0:1769074867.679723:0:67421:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:0.0:1769074867.679751:0:67421:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 3 00000080:00200000:0.0:1769074867.679758:0:67421:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), file ffff9b03462a0c00, open_flags 8041, mode 81b6 opened 0 00000080:00200000:0.0:1769074867.679770:0:67421:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=open|creat 00000002:00010000:0.0:1769074867.679812:0:67421:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x200000405:0x20b:0x0]) in obj [0x200000bd0:0x9:0x0], intent: open|creat flags 0100102 00010000:00010000:0.0:1769074867.679898:0:67421:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 3/0,1 mode: --/CW res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67421 initiator: MDT0 00010000:00010000:0.0:1769074867.679908:0:67421:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 3/0,1 mode: --/CW res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67421 initiator: MDT0 00000020:00100000:0.0:1769074867.679923:0:67421:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074867.679940:0:67421:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a38380 x1855007617037824/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/67824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074867.679951:0:67421:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 3/0,1 mode: --/CW res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67421 initiator: MDT0 00000100:00100000:0.0:1769074867.679977:0:67421:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67421:1855007617037824:192.168.204.160@tcp:101:dd.0 00000100:00100000:0.0:1769074867.680205:0:67421:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a780 going to sleep for 50 seconds 00000100:00100000:1.0:1769074867.686659:0:67421:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67421:1855007617037824:192.168.204.160@tcp:101:dd.0 00010000:00010000:1.0:1769074867.686695:0:67421:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 4/0,1 mode: --/CW res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074867.686732:0:67421:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 4/0,1 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074867.686740:0:67421:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 4/0,1 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074867.686754:0:67421:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 4/0,1 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074867.686766:0:67421:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 4/0,1 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074867.686780:0:67421:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 5/1,1 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074867.686787:0:67421:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 5/1,1 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074867.686794:0:67421:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00000002:00100000:1.0:1769074867.686805:0:67421:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=2b, status=0 req@ffff9b0365a38380 x1855007617037824/t8589944123(8589944123) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/1264 e 0 to 0 dl 1769074922 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1769074867.686819:0:67421:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0365a38380 x1855007617037824/t8589944123(8589944123) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/1264 e 0 to 0 dl 1769074922 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1769074867.686832:0:67421:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 656 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00000002:00010000:1.0:1769074867.686848:0:67421:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074867.686859:0:67421:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/522 (0 0) 00000080:00200000:1.0:1769074867.686867:0:67421:0:(namei.c:955:ll_intent_lock()) intent lock 3 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1769074867.686939:0:67421:0:(namei.c:151:ll_iget()) got inode: [0x200000405:0x20a:0x0](ffff9b0355ce3248): rc = 0 00000080:00200000:1.0:1769074867.686962:0:67421:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x20a:0x0]: layout version change: 4294967294 -> 6 00000080:00010000:1.0:1769074867.686970:0:67421:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c8b7 00000080:00010000:1.0:1769074867.686985:0:67421:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) 00000080:00200008:1.0:1769074867.687007:0:67421:0:(file.c:1001:ll_file_open()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248), flags 100101 00000002:00100000:1.0:1769074867.687037:0:67421:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0365a38380 x1855007617037824/t8589944123(8589944123) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1192/1264 e 0 to 0 dl 1769074922 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1769074867.687084:0:67421:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c8b7 from it ffff9b0348361000 00010000:00010000:1.0:1769074867.687090:0:67421:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074867.687105:0:67421:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074867.687142:0:67421:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248), inode mode 81a4 mask 42 00000080:00200008:0.0:1769074867.701075:0:67421:0:(file.c:2648:do_file_write_iter()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0], ppos: 0, count: 2097152 00000080:00200000:0.0:1769074867.701090:0:67421:0:(file.c:2672:do_file_write_iter()) switching to DIO 00000080:00200000:0.0:1769074867.701097:0:67421:0:(file.c:1994:ll_file_io_generic()) f204c.sanity-flr: write ppos: 0, bytes: 2097152 00000080:00200000:0.0:1769074867.701111:0:67421:0:(file.c:1759:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 0 00000020:00200000:0.0:1769074867.701121:0:67421:0:(cl_io.c:214:cl_io_rw_init()) header@0000000010960d79[0x4, 2, [0x200000405:0x20a:0x0] hash] 00000020:00200000:0.0:1769074867.701123:0:67421:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 2097152) 0 0 00000080:00200000:0.0:1769074867.701129:0:67421:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00020000:00200000:0.0:1769074867.701144:0:67421:0:(lov_io.c:385:lov_io_mirror_init()) [0x200000405:0x20a:0x0] need write intent for [0, 2097152) 00000020:00200000:0.0:1769074867.701149:0:67421:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074867.701156:0:67421:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 6 need write layout 1, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074867.701160:0:67421:0:(vvp_io.c:356:vvp_io_fini()) [0x200000405:0x20a:0x0] write layout, type 2 [0x0, 0x200000) 00000080:00010000:0.0:1769074867.701171:0:67421:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000405:0x20a:0x0](ffff9b0355ce3248) 00000002:00010000:0.0:1769074867.701188:0:67421:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20a:0x0]) in obj [0x200000405:0x20a:0x0], intent: layout flags 02 00010000:00010000:0.0:1769074867.701214:0:67421:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074867.701607:0:67421:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x20a:0x0]: losing layout lock 00010000:00010000:0.0:1769074867.701658:0:67421:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074867.701667:0:67421:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074867.701670:0:67421:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215200/0xb7d6bf619879c8b7 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd559f0 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074867.701709:0:67421:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(EX) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 3/0,1 mode: --/EX res: [0x200000405:0x20a:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67421 initiator: MDT0 00010000:00010000:0.0:1769074867.701716:0:67421:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 3/0,1 mode: --/EX res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67421 initiator: MDT0 00000020:00100000:0.0:1769074867.701729:0:67421:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074867.701741:0:67421:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3a300 x1855007617038080/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 376/1976 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074867.701750:0:67421:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 3/0,1 mode: --/EX res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67421 initiator: MDT0 00000100:00100000:0.0:1769074867.702581:0:67421:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67421:1855007617038080:192.168.204.160@tcp:101:dd.0 00000100:00100000:0.0:1769074867.702690:0:67421:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a0c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074867.717449:0:67421:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944124 gen 1 00000100:00100000:2.0:1769074867.717469:0:67421:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff9b0365a38380 x1855007617037824/t8589944123(8589944123) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1192/1264 e 0 to 0 dl 1769074922 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769074867.717482:0:67421:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67421:1855007617038080:192.168.204.160@tcp:101:dd.0 00010000:00010000:2.0:1769074867.717514:0:67421:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode CR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 4/0,1 mode: --/EX res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717525:0:67421:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 4/0,1 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717536:0:67421:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 4/0,1 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717546:0:67421:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 4/0,1 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717559:0:67421:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 5/1,1 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717566:0:67421:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(EX) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 5/1,1 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717572:0:67421:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000002:00100000:2.0:1769074867.717581:0:67421:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9b0365a3a300 x1855007617038080/t8589944124(8589944124) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 376/792 e 0 to 0 dl 1769074922 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074867.717603:0:67421:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000002:00010000:2.0:1769074867.717614:0:67421:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717626:0:67421:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 16 res 8589935621/522 (0 0) 00000080:00200000:2.0:1769074867.717632:0:67421:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200000405:0x20a:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1769074867.717648:0:67421:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c8be 00000080:00010000:2.0:1769074867.717656:0:67421:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c8be from it ffffb4364185bbb8 00010000:00010000:2.0:1769074867.717660:0:67421:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717672:0:67421:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074867.717688:0:67421:0:(file.c:2142:ll_file_io_generic()) f204c.sanity-flr: 2 io complete with rc: 0, result: 0, restart: 1 00000080:00200000:2.0:1769074867.717692:0:67421:0:(file.c:2149:ll_file_io_generic()) f204c.sanity-flr: restart write from ppos=0 bytes=2097152 retries=999 ret=0: rc = 0 00000080:00200000:2.0:1769074867.717699:0:67421:0:(file.c:1759:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 0 00000020:00200000:2.0:1769074867.717706:0:67421:0:(cl_io.c:214:cl_io_rw_init()) header@0000000010960d79[0x4, 2, [0x200000405:0x20a:0x0] hash] 00000020:00200000:2.0:1769074867.717708:0:67421:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 2097152) 0 0 00000080:00200000:2.0:1769074867.717714:0:67421:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 6 restore needed 0 00010000:00010000:2.0:1769074867.717732:0:67421:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717743:0:67421:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074867.717750:0:67421:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000405:0x20a:0x0](00000000020d7a38) being reconfigured ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717805:0:67421:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717813:0:67421:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074867.717828:0:67421:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074867.717840:0:67421:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074867.717842:0:67421:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074867.717848:0:67421:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:2.0:1769074867.717859:0:67421:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x40 00020000:00200000:2.0:1769074867.717861:0:67421:0:(lov_io.c:860:lov_io_iter_init()) component[5] flags 0x0 00000080:00200000:2.0:1769074867.717880:0:67421:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1769074867.717892:0:67421:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.717899:0:67421:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074867.717905:0:67421:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000405:0x20a:0x0](00000000020d7a38) being reconfigured ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074867.717920:0:67421:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074867.717925:0:67421:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074867.717927:0:67421:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074867.717928:0:67421:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:2.0:1769074867.717932:0:67421:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x40 00020000:00200000:2.0:1769074867.717933:0:67421:0:(lov_io.c:860:lov_io_iter_init()) component[5] flags 0x0 00000080:00200000:2.0:1769074867.718196:0:67421:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074867.718588:0:67421:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x20a:0x0]: layout version change: 4294967294 -> 7 00010000:00010000:2.0:1769074867.718596:0:67421:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074867.718604:0:67421:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00020000:00200000:2.0:1769074867.718615:0:67421:0:(lov_io.c:308:lov_io_mirror_write_intent()) [0x200000405:0x20a:0x0]there are 0 components to be staled to modify file extent [0x0, 0xa00000), iot: 2 00020000:00200000:2.0:1769074867.718619:0:67421:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074867.718622:0:67421:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:2.0:1769074867.718626:0:67421:0:(file.c:2072:ll_file_io_generic()) Range lock [0, 511] 00020000:00200000:2.0:1769074867.718634:0:67421:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 2097152) 2097152, 2097152 00020000:00200000:2.0:1769074867.718637:0:67421:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1769074867.718651:0:67421:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 511] 00000008:00010000:2.0:1769074867.718663:0:67421:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1769074867.718667:0:67421:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348d6be30, osc lock ffff9b0345126980, flags 0x0 00000080:00200000:2.0:1769074867.718677:0:67421:0:(vvp_io.c:1312:vvp_io_write_start()) f204c.sanity-flr: write [0, 2097152) append 0 append lockpos 0 00000002:00010000:2.0:1769074867.718705:0:67421:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x200000405:0x20a:0x0], intent: getxattr flags 0 00010000:00010000:2.0:1769074867.718744:0:67421:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 3/1,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67421 initiator: MDT0 00010000:00010000:2.0:1769074867.718752:0:67421:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 3/1,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67421 initiator: MDT0 00010000:00010000:2.0:1769074867.718761:0:67421:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 3/1,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67421 initiator: MDT0 00000100:00100000:2.0:1769074867.718790:0:67421:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67421:1855007617038208:192.168.204.160@tcp:101:dd.0 00000100:00100000:2.0:1769074867.718907:0:67421:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074867.728543:0:67421:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67421:1855007617038208:192.168.204.160@tcp:101:dd.0 00010000:00010000:2.0:1769074867.728572:0:67421:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a13 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074867.728590:0:67421:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55a13 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074867.728601:0:67421:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a13 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074867.728616:0:67421:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=80 disposition=0, status=0 req@ffff9b03482c6d80 x1855007617038208/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 568/1544 e 0 to 0 dl 1769074922 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074867.728629:0:67421:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a13 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074867.728644:0:67421:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/522 (0 0) 00000080:00200000:2.0:1769074867.728652:0:67421:0:(namei.c:955:ll_intent_lock()) intent lock 128 on i1 [0x200000405:0x20a:0x0] suppgids 0 -1: rc 0 00000080:00010000:2.0:1769074867.728679:0:67421:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c8c5 00000080:00010000:2.0:1769074867.728687:0:67421:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c8c5 from it ffffb4364185b8f8 00010000:00010000:2.0:1769074867.728691:0:67421:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55a13 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074867.728701:0:67421:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a13 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074867.728746:0:67421:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248), size=2097152 (max 603979776), offset=0=0x0, pages 512 (max 147456), lockless, parallel, hybrid 00000020:00200000:2.0:1769074867.729320:0:67421:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 512 items remaining. 00000080:00200000:2.0:1769074867.729337:0:67421:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 2097152 count 2097152 file_offset 2097152 00000020:00010000:2.0:1769074867.729349:0:67421:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0343c7e8a8 (ffff9b0353152a28/0) at cl_lock_release():245 00000020:00010000:2.0:1769074867.729352:0:67421:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0343c7e8a8 (ffff9b0353152a28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074867.729356:0:67421:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6be30 (ffff9b0342b7d368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074867.729360:0:67421:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0343c7e8a8 (ffff9b0353152a28/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074867.729362:0:67421:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6be30 (ffff9b0353152a28/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074867.729371:0:67421:0:(vvp_io.c:518:vvp_io_advance()) advancing 2097152 bytes 00000020:00200000:2.0:1769074867.729377:0:67421:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1769074867.729397:0:63495:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b0342c54cc0 00000008:00100000:2.0:1769074867.730353:0:63495:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b03482c6d80 - object 0x280000401:713 offset 0<>2097152 00000080:00200000:2.0:1769074867.810056:0:67421:0:(file.c:2109:ll_file_io_generic()) Range unlock [0, 511] 00000080:00200000:2.0:1769074867.810074:0:67421:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074867.810080:0:67421:0:(file.c:2142:ll_file_io_generic()) f204c.sanity-flr: 2 io complete with rc: 0, result: 2097152, restart: 0 00000080:00200000:2.0:1769074867.810097:0:67421:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x20a:0x0] updating i_size 2097152 i_blocks 4096 00000020:00200000:2.0:1769074867.810101:0:67421:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074867.810109:0:67421:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 2097152 00000080:00200008:2.0:1769074867.810149:0:67421:0:(file.c:424:ll_file_release()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248), flags 100001 00000002:00100000:2.0:1769074867.810221:0:67421:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0365a38380 x1855007617037824/t8589944123(8589944123) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1192/1264 e 0 to 0 dl 1769074922 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074867.810270:0:67421:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074867.810276:0:67421:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c6680 x1855007617038592/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074867.810303:0:67421:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67421:1855007617038592:192.168.204.160@tcp:35:dd.0 00000100:00100000:2.0:1769074867.810420:0:67421:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074867.815070:0:67421:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67421:1855007617038592:192.168.204.160@tcp:35:dd.0 00000002:00100000:2.0:1769074867.815111:0:67421:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0365a38380 x1855007617037824/t8589944123(8589944123) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1192/1264 e 0 to 0 dl 1769074922 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1769074869.720794:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074869.859087:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617038848:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074869.859190:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617039104:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074869.859226:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:0.0:1769074869.859600:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617038720:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074869.859727:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617038976:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074869.859748:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617039232:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074869.875343:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617038720:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074869.875369:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617038976:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074869.875380:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617039232:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074869.875392:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617038848:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074869.875401:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617039104:192.168.204.160@tcp:400:kworker.0 00010000:00010000:2.0:1769074870.739517:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074871.765363:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074873.811642:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074874.964985:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617039488:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074874.965091:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617039744:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074874.965129:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:0.0:1769074874.965136:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617039360:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074874.965161:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617039616:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074874.965180:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617039872:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074874.974502:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617039744:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074874.976311:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617039360:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074874.988309:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617039872:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074874.996149:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617039616:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074874.996164:0:2406:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944126 gen 1 00000100:00100000:1.0:1769074874.996169:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617039488:192.168.204.160@tcp:400:kworker.0 00010000:00010000:2.0:1769074876.884824:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074877.907548:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074878.931615:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1769074880.955840:0:67444:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074880.955870:0:67444:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074880.955887:0:67444:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074880.955903:0:67444:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074880.955917:0:67444:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074880.955925:0:67444:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074880.955929:0:67444:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:0.0:1769074880.955936:0:67444:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641ccfbf8 00010000:00010000:0.0:1769074880.955940:0:67444:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074880.955949:0:67444:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074880.955966:0:67444:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074880.955973:0:67444:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:0.0:1769074880.955979:0:67444:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:0.0:1769074880.955992:0:67444:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), file ffff9b035115af00, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1769074880.955999:0:67444:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=open 00000002:00010000:0.0:1769074880.956013:0:67444:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x9:0x0], intent: open flags 0304001 00010000:00010000:0.0:1769074880.956065:0:67444:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67444 initiator: MDT0 00010000:00010000:0.0:1769074880.956073:0:67444:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67444 initiator: MDT0 00000020:00100000:0.0:1769074880.956084:0:67444:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074880.956096:0:67444:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3bb80 x1855007617040768/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074880.956105:0:67444:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67444 initiator: MDT0 00000100:00100000:0.0:1769074880.956126:0:67444:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67444:1855007617040768:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074880.956312:0:67444:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a0c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074880.967645:0:67444:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67444:1855007617040768:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074880.967671:0:67444:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.967684:0:67444:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.967691:0:67444:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.967698:0:67444:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.967706:0:67444:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.967712:0:67444:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0342af4e00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074880.967726:0:67444:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.967734:0:67444:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.967741:0:67444:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.967747:0:67444:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074880.967753:0:67444:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.967760:0:67444:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af4e00/0xb7d6bf619879c8cc lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67444 initiator: MDT0 00000002:00100000:2.0:1769074880.967786:0:67444:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0365a3bb80 x1855007617040768/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074935 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074880.967802:0:67444:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074880.968294:0:67444:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074880.968317:0:67444:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074880.968332:0:67444:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074880.968349:0:67444:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074880.968357:0:67444:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074880.968363:0:67444:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074880.968367:0:67444:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074880.968373:0:67444:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641ccfbf8 00010000:00010000:2.0:1769074880.968377:0:67444:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074880.968385:0:67444:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074880.968398:0:67444:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074880.968406:0:67444:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:2.0:1769074880.968412:0:67444:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 44 00000080:00200008:2.0:1769074880.968421:0:67444:0:(file.c:1001:ll_file_open()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00000002:00010000:2.0:1769074880.968436:0:67444:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x209:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:2.0:1769074880.968488:0:67444:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.968495:0:67444:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67444 initiator: MDT0 00000020:00100000:2.0:1769074880.968506:0:67444:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074880.968514:0:67444:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c6a00 x1855007617041024/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074880.968522:0:67444:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67444 initiator: MDT0 00000100:00100000:2.0:1769074880.968543:0:67444:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67444:1855007617041024:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074880.968859:0:67444:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074880.977434:0:67444:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67444:1855007617041024:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074880.977458:0:67444:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.977472:0:67444:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.977478:0:67444:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.977485:0:67444:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.977493:0:67444:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.977499:0:67444:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860600 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074880.977504:0:67444:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.977511:0:67444:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.977518:0:67444:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.977524:0:67444:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074880.977530:0:67444:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67444 initiator: MDT0 00010000:00010000:2.0:1769074880.977539:0:67444:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8d3 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 67444 initiator: MDT0 00000002:00100000:2.0:1769074880.977559:0:67444:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9b03482c6a00 x1855007617041024/t8589944127(8589944127) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074935 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074880.977573:0:67444:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b03482c6a00 x1855007617041024/t8589944127(8589944127) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074935 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074880.977609:0:67444:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074880.977654:0:67444:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x209:0x0], updating i_size 4096 00000002:00100000:2.0:1769074880.977673:0:67444:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b03482c6a00 x1855007617041024/t8589944127(8589944127) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074935 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1769074880.977776:0:67444:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908) cmd=c0086915 arg=1f653d0 00000100:00100000:2.0:1769074880.977844:0:67444:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67444:1855007617041152:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074880.977928:0:67444:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00010000:00010000:2.0:1769074880.979637:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074880.990899:0:67444:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67444:1855007617041152:192.168.204.160@tcp:34:lfs.0 00000080:00200008:2.0:1769074880.991206:0:67444:0:(file.c:424:ll_file_release()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00000002:00100000:2.0:1769074880.991331:0:67444:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b03482c6a00 x1855007617041024/t8589944127(8589944127) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074935 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074880.991344:0:67444:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074880.991351:0:67444:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7800 x1855007617041408/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074880.991374:0:67444:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67444:1855007617041408:192.168.204.160@tcp:35:lfs.0 00000100:00100000:2.0:1769074880.991444:0:67444:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 11 seconds 00000100:00100000:2.0:1769074880.995517:0:67444:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67444:1855007617041408:192.168.204.160@tcp:35:lfs.0 00000002:00100000:2.0:1769074880.995557:0:67444:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b03482c6a00 x1855007617041024/t8589944127(8589944127) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074935 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074881.033949:0:67448:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074881.033985:0:67448:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.034006:0:67448:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.034023:0:67448:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.034032:0:67448:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.034043:0:67448:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074881.034046:0:67448:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074881.034054:0:67448:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641cd7bf8 00010000:00010000:2.0:1769074881.034058:0:67448:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.034068:0:67448:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.034085:0:67448:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074881.034093:0:67448:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:2.0:1769074881.034113:0:67448:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:2.0:1769074881.034127:0:67448:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), file ffff9b0372f5ec00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074881.034134:0:67448:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=open 00000002:00010000:2.0:1769074881.034151:0:67448:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x9:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074881.034216:0:67448:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8da lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67448 initiator: MDT0 00010000:00010000:2.0:1769074881.034224:0:67448:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8da lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67448 initiator: MDT0 00000020:00100000:2.0:1769074881.034254:0:67448:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074881.034266:0:67448:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7800 x1855007617041664/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074881.034276:0:67448:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c8da lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67448 initiator: MDT0 00000100:00100000:2.0:1769074881.034310:0:67448:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67448:1855007617041664:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074881.034495:0:67448:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000080:00200008:2.0:1769074881.147789:0:67453:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074881.147800:0:67453:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074881.147826:0:67453:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.147845:0:67453:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.147861:0:67453:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.147869:0:67453:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.147876:0:67453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074881.147880:0:67453:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074881.147901:0:67453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641ccfcb8 00010000:00010000:2.0:1769074881.147905:0:67453:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.147920:0:67453:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.148049:0:67453:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074881.148059:0:67453:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.148066:0:67453:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.148075:0:67453:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.148081:0:67453:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.148085:0:67453:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074881.148087:0:67453:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074881.148090:0:67453:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641ccfbb0 00010000:00010000:2.0:1769074881.148520:0:67453:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.148531:0:67453:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.148542:0:67453:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:2.0:1769074881.148551:0:67453:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:2.0:1769074881.148570:0:67453:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffeafbab60c 00000080:00200008:2.0:1769074881.148584:0:67453:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:1.0:1769074881.151313:0:67455:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074881.151341:0:67455:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074881.151358:0:67455:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.151372:0:67455:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074881.151380:0:67455:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074881.151388:0:67455:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074881.151391:0:67455:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:1.0:1769074881.151398:0:67455:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641c77bf8 00010000:00010000:1.0:1769074881.151402:0:67455:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074881.151410:0:67455:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074881.151426:0:67455:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074881.151432:0:67455:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:1.0:1769074881.151562:0:67455:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:1.0:1769074881.151574:0:67455:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), file ffff9b034583af00, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1769074881.151581:0:67455:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=open 00000002:00010000:1.0:1769074881.151605:0:67455:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x9:0x0], intent: open flags 0304001 00010000:00010000:1.0:1769074881.151658:0:67455:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67455 initiator: MDT0 00010000:00010000:1.0:1769074881.151665:0:67455:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67455 initiator: MDT0 00000020:00100000:1.0:1769074881.151678:0:67455:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074881.151691:0:67455:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ce8700 x1855007617042304/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074881.151700:0:67455:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67455 initiator: MDT0 00000100:00100000:1.0:1769074881.151721:0:67455:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67455:1855007617042304:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074881.152194:0:67455:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372e6d6c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074881.164803:0:67455:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67455:1855007617042304:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074881.164827:0:67455:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67455 initiator: MDT0 00010000:00010000:2.0:1769074881.164842:0:67455:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67455 initiator: MDT0 00010000:00010000:2.0:1769074881.164848:0:67455:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67455 initiator: MDT0 00010000:00010000:2.0:1769074881.164854:0:67455:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67455 initiator: MDT0 00010000:00010000:2.0:1769074881.164861:0:67455:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67455 initiator: MDT0 00010000:00010000:2.0:1769074881.164866:0:67455:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919c200 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074881.164870:0:67455:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67455 initiator: MDT0 00010000:00010000:2.0:1769074881.164876:0:67455:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67455 initiator: MDT0 00010000:00010000:2.0:1769074881.164895:0:67455:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 67455 initiator: MDT0 00010000:00010000:2.0:1769074881.164901:0:67455:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074881.164906:0:67455:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67455 initiator: MDT0 00010000:00010000:2.0:1769074881.164912:0:67455:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879c8e8 lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67455 initiator: MDT0 00000002:00100000:2.0:1769074881.164937:0:67455:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0342ce8700 x1855007617042304/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074936 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074881.164950:0:67455:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074881.165445:0:67455:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074881.165469:0:67455:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.165482:0:67455:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.165494:0:67455:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.165501:0:67455:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.165508:0:67455:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074881.165512:0:67455:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074881.165517:0:67455:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641c77bf8 00010000:00010000:2.0:1769074881.165521:0:67455:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.165528:0:67455:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.165541:0:67455:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074881.165549:0:67455:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:2.0:1769074881.165556:0:67455:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 44 00000080:00200008:2.0:1769074881.165564:0:67455:0:(file.c:1001:ll_file_open()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00000080:00200080:2.0:1769074881.165615:0:67455:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908) cmd=c1406916 arg=1c64a10 00000100:00100000:2.0:1769074881.165676:0:67455:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67455:1855007617042432:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074881.166071:0:67455:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55cc0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074881.181030:0:67455:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67455:1855007617042432:192.168.204.160@tcp:34:lfs.0 00000080:00200008:2.0:1769074881.181207:0:67455:0:(file.c:424:ll_file_release()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00010000:00010000:2.0:1769074881.181255:0:67455:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b800/0xb7d6bf619879c8e1 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x209:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a75 expref: -99 pid: 67448 timeout: 0 lvb_type: 0 00000100:00100000:0.0:1769074881.216570:0:67460:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67460:1855007617042688:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074881.216599:0:67460:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c8ef lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67460 initiator: MDT0 00010000:00010000:0.0:1769074881.216612:0:67460:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c8ef lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67460 initiator: MDT0 00010000:00010000:0.0:1769074881.216619:0:67460:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c8ef lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67460 initiator: MDT0 00010000:00010000:0.0:1769074881.216626:0:67460:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c8ef lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67460 initiator: MDT0 00010000:00010000:0.0:1769074881.216633:0:67460:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c8ef lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67460 initiator: MDT0 00010000:00010000:0.0:1769074881.216638:0:67460:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035005a800 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074881.216643:0:67460:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c8ef lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67460 initiator: MDT0 00010000:00010000:0.0:1769074881.216650:0:67460:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c8ef lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67460 initiator: MDT0 00010000:00010000:0.0:1769074881.216656:0:67460:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c8ef lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 67460 initiator: MDT0 00010000:00010000:0.0:1769074881.216663:0:67460:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074881.216668:0:67460:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c8ef lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67460 initiator: MDT0 00010000:00010000:0.0:1769074881.216674:0:67460:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c8ef lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67460 initiator: MDT0 00000002:00100000:0.0:1769074881.216698:0:67460:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0352a1b100 x1855007617042688/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074936 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074881.216731:0:67460:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1769074881.216857:0:67460:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074881.217060:0:67460:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074881.217075:0:67460:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074881.217087:0:67460:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074881.217094:0:67460:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074881.217099:0:67460:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074881.217103:0:67460:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:0.0:1769074881.217108:0:67460:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641ccfbf8 00010000:00010000:0.0:1769074881.217112:0:67460:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074881.217119:0:67460:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074881.217130:0:67460:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074881.217137:0:67460:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:0.0:1769074881.217143:0:67460:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 44 00000080:00200008:0.0:1769074881.217151:0:67460:0:(file.c:1001:ll_file_open()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00000080:00200080:0.0:1769074881.217190:0:67460:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908) cmd=c0086915 arg=e483d0 00000100:00100000:0.0:1769074881.217306:0:67460:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67460:1855007617042816:192.168.204.160@tcp:34:lfs.0 00000100:00100000:0.0:1769074881.217372:0:67460:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a780 going to sleep for 50 seconds 00000100:00100000:0.0:1769074881.227146:0:67460:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67460:1855007617042816:192.168.204.160@tcp:34:lfs.0 00000080:00200008:0.0:1769074881.227401:0:67460:0:(file.c:424:ll_file_release()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00010000:00010000:0.0:1769074881.227444:0:67460:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b800/0xb7d6bf619879c8e1 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x209:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a75 expref: -99 pid: 67448 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.248319:0:67463:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074881.248352:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.248371:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.248387:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.248396:0:67463:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.248629:0:67463:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074881.248633:0:67463:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074881.248641:0:67463:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641cb7b58 00010000:00010000:2.0:1769074881.248645:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.248655:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.248672:0:67463:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074881.248679:0:67463:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=25 00000080:00200000:2.0:1769074881.248686:0:67463:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:2.0:1769074881.248698:0:67463:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), flags=9 00000080:00200000:2.0:1769074881.248702:0:67463:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=getattr 00000002:00010000:2.0:1769074881.248790:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x9:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.248844:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67463 initiator: MDT0 00010000:00010000:2.0:1769074881.248852:0:67463:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67463 initiator: MDT0 00010000:00010000:2.0:1769074881.248862:0:67463:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67463 initiator: MDT0 00000100:00100000:2.0:1769074881.248963:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617042944:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074881.249116:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55cc0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074881.280546:0:67463:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617042944:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074881.280576:0:67463:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.280605:0:67463:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.280613:0:67463:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.280634:0:67463:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.280645:0:67463:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.280658:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.280666:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.280673:0:67463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074881.280685:0:67463:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b03482c6a00 x1855007617042944/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/1032 e 0 to 0 dl 1769074936 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074881.280696:0:67463:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1769074881.280721:0:67463:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.280733:0:67463:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/522 (0 0) 00000080:00200000:2.0:1769074881.280739:0:67463:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074881.280792:0:67463:0:(namei.c:151:ll_iget()) got inode: [0x200000405:0x20a:0x0](ffff9b0355ce3248): rc = 0 00000080:00200000:2.0:1769074881.280808:0:67463:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x20a:0x0]: layout version change: 7 -> 7 00000080:00010000:2.0:1769074881.280813:0:67463:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c8f6 00000080:00010000:2.0:1769074881.280824:0:67463:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) 00000080:00010000:2.0:1769074881.280827:0:67463:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c8f6 from it ffffb43641cb7b20 00010000:00010000:2.0:1769074881.280831:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.280839:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200008:2.0:1769074881.280873:0:67463:0:(file.c:6252:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074881.281045:0:67463:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248),name=f204c.sanity-flr 00000002:00010000:2.0:1769074881.281073:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20a:0x0]) in obj [0x200000bd0:0x9:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.281083:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.281109:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074881.281117:0:67463:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x9:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.281122:0:67463:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) 00000080:00010000:2.0:1769074881.281125:0:67463:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c8f6 00000080:00010000:2.0:1769074881.281129:0:67463:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c8f6 from it ffffb43641cb7cb8 00010000:00010000:2.0:1769074881.281132:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.281138:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074881.281157:0:67463:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074881.281168:0:67463:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074881.281171:0:67463:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074881.281175:0:67463:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x20a:0x0] 00000008:00010000:2.0:1769074881.281198:0:67463:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074881.281202:0:67463:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348d6b9b0, osc lock ffff9b0345126980, flags 0x400000400001002 00010000:00010000:2.0:1769074881.281364:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c8fd lrc: 3/1,0 mode: --/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.281373:0:67463:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c8fd lrc: 3/1,0 mode: --/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.281384:0:67463:0:(ldlm_request.c:1054:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff9b03482c6a00 x1855007617043072/t0(0) o101->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:2.0:1769074881.281586:0:67463:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1769074881.294019:0:67463:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x20a:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1769074881.294030:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_release():245 00000020:00010000:2.0:1769074881.294034:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074881.294040:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6b9b0 (ffff9b0342b7d368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074881.294046:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074881.294048:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6b9b0 (ffff9b0353152a28/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074881.294066:0:67463:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074881.294139:0:67463:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074881.294164:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.294180:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.294194:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.294201:0:67463:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.294209:0:67463:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074881.294212:0:67463:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074881.294219:0:67463:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641cb7bf8 00010000:00010000:2.0:1769074881.294222:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.294229:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074881.294262:0:67463:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074881.294270:0:67463:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=25 00000080:00200000:2.0:1769074881.294287:0:67463:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:2.0:1769074881.294291:0:67463:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204c.sanity-flr, flags=265 00000080:00200000:2.0:1769074881.294296:0:67463:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248), inode mode 81a4 mask 46 00000080:00200008:2.0:1769074881.294305:0:67463:0:(file.c:1001:ll_file_open()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248), flags 154402 00000002:00010000:2.0:1769074881.294319:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20a:0x0]) in obj [0x200000405:0x20a:0x0], intent: open flags 0100240114403 00010000:00010000:2.0:1769074881.294377:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 67463 initiator: MDT0 00010000:00010000:2.0:1769074881.294385:0:67463:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 67463 initiator: MDT0 00000020:00100000:2.0:1769074881.294397:0:67463:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074881.294613:0:67463:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7100 x1855007617043328/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074881.294624:0:67463:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT flags: 0x0 pid: 67463 initiator: MDT0 00000100:00100000:2.0:1769074881.294648:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617043328:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074881.294912:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55cc0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074881.304337:0:67463:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617043328:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074881.304368:0:67463:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 4/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.304380:0:67463:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.304392:0:67463:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.304545:0:67463:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.304561:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.304569:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.304575:0:67463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074881.304586:0:67463:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9b03482c7100 x1855007617043328/t8589944130(8589944130) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/1056 e 0 to 0 dl 1769074936 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074881.304600:0:67463:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b03482c7100 x1855007617043328/t8589944130(8589944130) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/1056 e 0 to 0 dl 1769074936 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074881.304613:0:67463:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1769074881.304802:0:67463:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.304819:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.304831:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.304843:0:67463:0:(ldlm_lock.c:947:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.304849:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.304855:0:67463:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879c904 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0x7fe2323adfd55b16 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.305021:0:67463:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:2.0:1769074881.305030:0:67463:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000405:0x20a:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074881.305102:0:67463:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x20a:0x0]: layout version change: 7 -> 7 00000080:00010000:2.0:1769074881.305108:0:67463:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c8f6 00000080:00010000:2.0:1769074881.305114:0:67463:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c8f6 from it ffffb43641cb7bf0 00010000:00010000:2.0:1769074881.305118:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.305125:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000002:00100000:2.0:1769074881.305151:0:67463:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b03482c7100 x1855007617043328/t8589944130(8589944130) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 968/1056 e 0 to 0 dl 1769074936 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074881.305342:0:67463:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248), inode mode 81a4 mask 4 00000080:00200000:2.0:1769074881.305351:0:67463:0:(xattr.c:725:ll_xattr_get()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248), xattr lov 00000080:00200008:2.0:1769074881.305379:0:67463:0:(file.c:6252:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074881.305385:0:67463:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248),name=f204c.sanity-flr 00000002:00010000:2.0:1769074881.305420:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20a:0x0]) in obj [0x200000bd0:0x9:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074881.305432:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.305441:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074881.305447:0:67463:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x9:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074881.305453:0:67463:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) 00000080:00010000:2.0:1769074881.305456:0:67463:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c8f6 00000080:00010000:2.0:1769074881.305460:0:67463:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c8f6 from it ffffb43641cb7cd8 00010000:00010000:2.0:1769074881.305463:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.305469:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074881.305488:0:67463:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074881.305501:0:67463:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074881.305504:0:67463:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074881.305508:0:67463:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x20a:0x0] 00000008:00010000:2.0:1769074881.305528:0:67463:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074881.305531:0:67463:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348d6b9b0, osc lock ffff9b0345126980, flags 0x400000400001002 00010000:00010000:2.0:1769074881.305547:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c8fd lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b08 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.305557:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c8fd lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b08 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074881.305569:0:67463:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074881.305573:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c8fd lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b08 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.305581:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c8fd lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b08 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074881.305590:0:67463:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074881.305602:0:67463:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x20a:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:2.0:1769074881.305606:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_release():245 00000020:00010000:2.0:1769074881.305608:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074881.305612:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6b9b0 (ffff9b0342b7d368/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074881.305615:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074881.305618:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6b9b0 (ffff9b0353152a28/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074881.305639:0:67463:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:2.0:1769074881.305675:0:67463:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248) cmd=c00c66f3 arg=b802a0 00000002:00010000:2.0:1769074881.305690:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20a:0x0]) in obj [0x200000405:0x20a:0x0], intent: open flags 05004040000002 00010000:00010000:2.0:1769074881.306229:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c90b lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x0/0x0 rrc: 6 type: IBT flags: 0x10000000000000 pid: 67463 initiator: MDT0 00010000:00010000:2.0:1769074881.306340:0:67463:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x81000000001000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c90b lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x1/0x0 rrc: 6 type: IBT flags: 0x0 pid: 67463 initiator: MDT0 00000020:00100000:2.0:1769074881.306352:0:67463:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074881.306362:0:67463:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c5f80 x1855007617043584/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074881.306371:0:67463:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c90b lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x1/0x0 rrc: 6 type: IBT flags: 0x81000000000000 pid: 67463 initiator: MDT0 00000100:00100000:2.0:1769074881.306394:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617043584:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074881.306892:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074881.308265:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617043712:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074881.311807:0:2408:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074881.311825:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617043840:192.168.204.160@tcp:1:lfs.0 00000100:00100000:2.0:1769074881.321123:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617043712:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074881.323955:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074881.324038:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625253888 00000100:00100000:0.0:1769074881.324048:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625253888 00000100:00100000:0.0:1769074881.324108:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8950 00000100:00100000:0.0:1769074881.324117:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8396:x1855007625253888:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074881.324131:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879c8fd lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55b08 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.324201:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074881.324207:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8396:x1855007625253888:12345-192.168.204.160@tcp:104: Request processed in 92us (264us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074881.324214:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8950 00000100:00100000:2.0:1769074881.324983:0:2409:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074881.324999:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617043968:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074881.337735:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617043968:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074881.338031:0:67463:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074881.338057:0:67463:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x20a:0x0]: losing layout lock 00010000:00010000:2.0:1769074881.338076:0:67463:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074881.338086:0:67463:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x20a:0x0]: losing layout lock 00010000:00010000:2.0:1769074881.338130:0:67463:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.338138:0:67463:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 1/0,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.338145:0:67463:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1769074881.338148:0:67463:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c8f6 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55b01 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.338162:0:67463:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215400/0xb7d6bf619879c8be lrc: 0/0,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55a05 expref: -99 pid: 67421 timeout: 0 lvb_type: 3 00000020:00100000:2.0:1769074881.338177:0:67463:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074881.338185:0:67463:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7b80 x1855007617044224/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 472/440 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074881.338210:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617044224:192.168.204.160@tcp:36:lfs.0 00000100:00100000:2.0:1769074881.338359:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074881.363707:0:67463:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944132 gen 1 00000100:00100000:2.0:1769074881.363738:0:67463:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff9b0352a19880 x1855007617041920/t8589944129(8589944129) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074936 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769074881.363749:0:67463:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff9b03482c7100 x1855007617043328/t8589944130(8589944130) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 968/1056 e 0 to 0 dl 1769074936 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769074881.363758:0:67463:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff9b03482c5f80 x1855007617043584/t8589944131(8589944131) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 968/1056 e 0 to 0 dl 1769074936 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769074881.363767:0:67463:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617044224:192.168.204.160@tcp:36:lfs.0 00010000:00010000:2.0:1769074881.363816:0:67463:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 31 res 8589935621/522 (0 0) 00000080:00010000:2.0:1769074881.363829:0:67463:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200000405:0x20a:0x0](ffff9b0355ce3248) 00000002:00010000:2.0:1769074881.363840:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20a:0x0]) in obj [0x200000405:0x20a:0x0], intent: layout flags 0 00010000:00010000:2.0:1769074881.363897:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 67463 initiator: MDT0 00010000:00010000:2.0:1769074881.363907:0:67463:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67463 initiator: MDT0 00010000:00010000:2.0:1769074881.363917:0:67463:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 67463 initiator: MDT0 00000100:00100000:2.0:1769074881.363941:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617044352:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074881.364038:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074881.392662:0:67463:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617044352:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074881.392696:0:67463:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.392724:0:67463:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000002:00100000:2.0:1769074881.392739:0:67463:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9b03482c7b80 x1855007617044352/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 376/792 e 0 to 0 dl 1769074936 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074881.392750:0:67463:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000002:00010000:2.0:1769074881.392764:0:67463:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.392777:0:67463:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 16 res 8589935621/522 (0 0) 00000080:00200000:2.0:1769074881.392783:0:67463:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x200000405:0x20a:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1769074881.392802:0:67463:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c912 00000080:00010000:2.0:1769074881.392809:0:67463:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c912 from it ffffb43641cb7c70 00010000:00010000:2.0:1769074881.392813:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.392823:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.392851:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.392861:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074881.392868:0:67463:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000405:0x20a:0x0](00000000020d7a38) being reconfigured ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.393077:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.393087:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074881.393101:0:67463:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074881.393113:0:67463:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074881.393117:0:67463:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074881.393124:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074881.393135:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1769074881.393155:0:67463:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1769074881.393168:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.393176:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074881.393184:0:67463:0:(file.c:6914:ll_layout_lock_set()) ### file [0x200000405:0x20a:0x0](00000000020d7a38) being reconfigured ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074881.393206:0:67463:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 7 restore needed 0 00020000:00200000:2.0:1769074881.393211:0:67463:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074881.393213:0:67463:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074881.393215:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074881.393219:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:2.0:1769074881.393225:0:67463:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074881.393348:0:67463:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x20a:0x0]: layout version change: 4294967294 -> 8 00010000:00010000:2.0:1769074881.393353:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.393361:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200080:2.0:1769074881.393382:0:67463:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248) cmd=66f4 arg=ffffffffffffffff 00000080:00200008:2.0:1769074881.393395:0:67463:0:(file.c:6252:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074881.394386:0:67463:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248),name=f204c.sanity-flr 00000002:00010000:0.0:1769074881.394421:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20a:0x0]) in obj [0x200000405:0x20a:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074881.394433:0:67463:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 30 res 8589935621/522 (0 0) 00010000:00010000:0.0:1769074881.394481:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 67463 initiator: MDT0 00010000:00010000:0.0:1769074881.394488:0:67463:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 67463 initiator: MDT0 00010000:00010000:0.0:1769074881.394497:0:67463:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 67463 initiator: MDT0 00000100:00100000:0.0:1769074881.394521:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617044480:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074881.394631:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ad80 going to sleep for 50 seconds 00000080:00200000:0.0:1769074881.427359:0:67463:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x20a:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1769074881.427366:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03513e76c0 (ffff9b0342ca8ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074881.427369:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03513e76c0 (ffff9b0342ca8ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074881.427375:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359274830 (ffff9b0342ca85a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074881.427379:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03513e76c0 (ffff9b0342ca8ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074881.427382:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359274830 (ffff9b0342ca8ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074881.427396:0:67463:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1769074881.482774:0:67463:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248) cmd=400866b1 arg=1 00000080:00200008:1.0:1769074881.482793:0:67463:0:(file.c:6252:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074881.482801:0:67463:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248),name=f204c.sanity-flr 00000002:00010000:1.0:1769074881.482826:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20a:0x0]) in obj [0x200000405:0x20a:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074881.482845:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074881.482862:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074881.482871:0:67463:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000405:0x20a:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074881.482881:0:67463:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) 00000080:00010000:1.0:1769074881.482884:0:67463:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c919 00000080:00010000:1.0:1769074881.482891:0:67463:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c919 from it ffffb43641cb7cd8 00010000:00010000:1.0:1769074881.482895:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074881.482904:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074881.482931:0:67463:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1769074881.482944:0:67463:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074881.482948:0:67463:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1769074881.482952:0:67463:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x20a:0x0] 00000008:00010000:1.0:1769074881.482976:0:67463:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074881.482980:0:67463:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360af67b0, osc lock ffff9b0345a09300, flags 0x400000400001002 00010000:00010000:1.0:1769074881.482999:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.483010:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074881.483022:0:67463:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074881.483026:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.483033:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074881.483044:0:67463:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074881.483054:0:67463:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x20a:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:1.0:1769074881.483058:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0356961da8 (ffff9b035121f638/0) at cl_lock_release():245 00000020:00010000:1.0:1769074881.483061:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0356961da8 (ffff9b035121f638/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074881.483064:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360af67b0 (ffff9b035121f758/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074881.483068:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0356961da8 (ffff9b035121f638/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074881.483070:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360af67b0 (ffff9b035121f638/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074881.483081:0:67463:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074881.483112:0:67463:0:(file.c:5185:ll_file_seek()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0], offset: 0, type: SEEK_DATA 00000080:00200000:1.0:1769074881.483119:0:67463:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 8 restore needed 0 00020000:00200000:1.0:1769074881.483124:0:67463:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074881.483126:0:67463:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1769074881.483131:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1769074881.483136:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1769074881.483150:0:67463:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x20a:0x0] updating i_size 2097152 i_blocks 4096 00000080:00200000:1.0:1769074881.483158:0:67463:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074881.483164:0:67463:0:(file.c:1759:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 1 00000080:00200000:1.0:1769074881.483166:0:67463:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074881.483170:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1769074881.483173:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1769074881.483176:0:67463:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [0, 18446744073709551615] 00000008:00010000:1.0:1769074881.483182:0:67463:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1769074881.483185:0:67463:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360af67b0, osc lock ffff9b0345a09300, flags 0x0 00010000:00010000:1.0:1769074881.483193:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.483202:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.483211:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.483218:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.483225:0:67463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769074881.489214:0:67463:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x20a:0x0] updating i_size 2097152 i_blocks 4096 00000100:00100000:1.0:1769074881.489339:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769074881.489365:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617044736:192.168.204.160@tcp:23:lfs.0 00000100:00100000:1.0:1769074881.494912:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617044736:192.168.204.160@tcp:23:lfs.0 00000100:00100000:2.0:1769074881.495596:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617044864:192.168.204.160@tcp:23:lfs.0 00000100:00100000:2.0:1769074881.509029:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617044864:192.168.204.160@tcp:23:lfs.0 00000100:00100000:2.0:1769074881.510760:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617044992:192.168.204.160@tcp:3:lfs.0 00000008:00010000:2.0:1769074881.603362:0:2409:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b035349f240 00000020:00200000:2.0:1769074881.603375:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 511 items remaining. 00000020:00200000:2.0:1769074881.603379:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 510 items remaining. 00000020:00200000:2.0:1769074881.603381:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 509 items remaining. 00000020:00200000:2.0:1769074881.603383:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 508 items remaining. 00000020:00200000:2.0:1769074881.603385:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 507 items remaining. 00000020:00200000:2.0:1769074881.603387:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 506 items remaining. 00000020:00200000:2.0:1769074881.603389:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 505 items remaining. 00000020:00200000:2.0:1769074881.603391:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 504 items remaining. 00000020:00200000:2.0:1769074881.603392:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 503 items remaining. 00000020:00200000:2.0:1769074881.603394:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 502 items remaining. 00000020:00200000:2.0:1769074881.603395:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 501 items remaining. 00000020:00200000:2.0:1769074881.603396:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 500 items remaining. 00000020:00200000:2.0:1769074881.603397:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 499 items remaining. 00000020:00200000:2.0:1769074881.603399:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 498 items remaining. 00000020:00200000:2.0:1769074881.603400:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 497 items remaining. 00000020:00200000:2.0:1769074881.603402:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 496 items remaining. 00000020:00200000:2.0:1769074881.603403:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 495 items remaining. 00000020:00200000:2.0:1769074881.603405:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 494 items remaining. 00000020:00200000:2.0:1769074881.603407:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 493 items remaining. 00000020:00200000:2.0:1769074881.603408:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 492 items remaining. 00000020:00200000:2.0:1769074881.603410:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 491 items remaining. 00000020:00200000:2.0:1769074881.603411:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 490 items remaining. 00000020:00200000:2.0:1769074881.603413:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 489 items remaining. 00000020:00200000:2.0:1769074881.603414:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 488 items remaining. 00000020:00200000:2.0:1769074881.603416:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 487 items remaining. 00000020:00200000:2.0:1769074881.603418:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 486 items remaining. 00000020:00200000:2.0:1769074881.603419:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 485 items remaining. 00000020:00200000:2.0:1769074881.603421:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 484 items remaining. 00000020:00200000:2.0:1769074881.603422:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 483 items remaining. 00000020:00200000:2.0:1769074881.603423:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 482 items remaining. 00000020:00200000:2.0:1769074881.603425:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 481 items remaining. 00000020:00200000:2.0:1769074881.603426:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 480 items remaining. 00000020:00200000:2.0:1769074881.603427:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 479 items remaining. 00000020:00200000:2.0:1769074881.603429:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 478 items remaining. 00000020:00200000:2.0:1769074881.603430:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 477 items remaining. 00000020:00200000:2.0:1769074881.603431:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 476 items remaining. 00000020:00200000:2.0:1769074881.603433:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 475 items remaining. 00000020:00200000:2.0:1769074881.603434:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 474 items remaining. 00000020:00200000:2.0:1769074881.603436:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 473 items remaining. 00000020:00200000:2.0:1769074881.603438:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 472 items remaining. 00000020:00200000:2.0:1769074881.603439:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 471 items remaining. 00000020:00200000:2.0:1769074881.603441:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 470 items remaining. 00000020:00200000:2.0:1769074881.603443:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 469 items remaining. 00000020:00200000:2.0:1769074881.603444:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 468 items remaining. 00000020:00200000:2.0:1769074881.603445:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 467 items remaining. 00000020:00200000:2.0:1769074881.603446:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 466 items remaining. 00000020:00200000:2.0:1769074881.603448:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 465 items remaining. 00000020:00200000:2.0:1769074881.603450:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 464 items remaining. 00000020:00200000:2.0:1769074881.603451:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 463 items remaining. 00000020:00200000:2.0:1769074881.603452:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 462 items remaining. 00000020:00200000:2.0:1769074881.603454:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 461 items remaining. 00000020:00200000:2.0:1769074881.603455:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 460 items remaining. 00000020:00200000:2.0:1769074881.603457:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 459 items remaining. 00000020:00200000:2.0:1769074881.603458:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 458 items remaining. 00000020:00200000:2.0:1769074881.603459:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 457 items remaining. 00000020:00200000:2.0:1769074881.603460:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 456 items remaining. 00000020:00200000:2.0:1769074881.603462:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 455 items remaining. 00000020:00200000:2.0:1769074881.603463:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 454 items remaining. 00000020:00200000:2.0:1769074881.603464:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 453 items remaining. 00000020:00200000:2.0:1769074881.603465:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 452 items remaining. 00000020:00200000:2.0:1769074881.603467:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 451 items remaining. 00000020:00200000:2.0:1769074881.603468:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 450 items remaining. 00000020:00200000:2.0:1769074881.603469:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 449 items remaining. 00000020:00200000:2.0:1769074881.603471:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 448 items remaining. 00000020:00200000:2.0:1769074881.603472:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 447 items remaining. 00000020:00200000:2.0:1769074881.603473:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 446 items remaining. 00000020:00200000:2.0:1769074881.603475:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 445 items remaining. 00000020:00200000:2.0:1769074881.603476:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 444 items remaining. 00000020:00200000:2.0:1769074881.603478:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 443 items remaining. 00000020:00200000:2.0:1769074881.603479:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 442 items remaining. 00000020:00200000:2.0:1769074881.603481:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 441 items remaining. 00000020:00200000:2.0:1769074881.603483:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 440 items remaining. 00000020:00200000:2.0:1769074881.603484:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 439 items remaining. 00000020:00200000:2.0:1769074881.603486:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 438 items remaining. 00000020:00200000:2.0:1769074881.603487:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 437 items remaining. 00000020:00200000:2.0:1769074881.603489:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 436 items remaining. 00000020:00200000:2.0:1769074881.603490:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 435 items remaining. 00000020:00200000:2.0:1769074881.603492:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 434 items remaining. 00000020:00200000:2.0:1769074881.603493:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 433 items remaining. 00000020:00200000:2.0:1769074881.603495:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 432 items remaining. 00000020:00200000:2.0:1769074881.603496:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 431 items remaining. 00000020:00200000:2.0:1769074881.603498:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 430 items remaining. 00000020:00200000:2.0:1769074881.603499:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 429 items remaining. 00000020:00200000:2.0:1769074881.603501:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 428 items remaining. 00000020:00200000:2.0:1769074881.603502:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 427 items remaining. 00000020:00200000:2.0:1769074881.603503:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 426 items remaining. 00000020:00200000:2.0:1769074881.603504:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 425 items remaining. 00000020:00200000:2.0:1769074881.603505:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 424 items remaining. 00000020:00200000:2.0:1769074881.603507:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 423 items remaining. 00000020:00200000:2.0:1769074881.603508:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 422 items remaining. 00000020:00200000:2.0:1769074881.603510:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 421 items remaining. 00000020:00200000:2.0:1769074881.603512:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 420 items remaining. 00000020:00200000:2.0:1769074881.603513:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 419 items remaining. 00000020:00200000:2.0:1769074881.603515:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 418 items remaining. 00000020:00200000:2.0:1769074881.603517:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 417 items remaining. 00000020:00200000:2.0:1769074881.603518:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 416 items remaining. 00000020:00200000:2.0:1769074881.603519:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 415 items remaining. 00000020:00200000:2.0:1769074881.603521:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 414 items remaining. 00000020:00200000:2.0:1769074881.603523:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 413 items remaining. 00000020:00200000:2.0:1769074881.603525:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 412 items remaining. 00000020:00200000:2.0:1769074881.603526:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 411 items remaining. 00000020:00200000:2.0:1769074881.603528:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 410 items remaining. 00000020:00200000:2.0:1769074881.603529:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 409 items remaining. 00000020:00200000:2.0:1769074881.603531:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 408 items remaining. 00000020:00200000:2.0:1769074881.603532:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 407 items remaining. 00000020:00200000:2.0:1769074881.603534:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 406 items remaining. 00000020:00200000:2.0:1769074881.603535:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 405 items remaining. 00000020:00200000:2.0:1769074881.603537:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 404 items remaining. 00000020:00200000:2.0:1769074881.603538:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 403 items remaining. 00000020:00200000:2.0:1769074881.603539:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 402 items remaining. 00000020:00200000:2.0:1769074881.603541:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 401 items remaining. 00000020:00200000:2.0:1769074881.603542:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 400 items remaining. 00000020:00200000:2.0:1769074881.603543:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 399 items remaining. 00000020:00200000:2.0:1769074881.603544:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 398 items remaining. 00000020:00200000:2.0:1769074881.603546:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 397 items remaining. 00000020:00200000:2.0:1769074881.603547:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 396 items remaining. 00000020:00200000:2.0:1769074881.603548:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 395 items remaining. 00000020:00200000:2.0:1769074881.603549:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 394 items remaining. 00000020:00200000:2.0:1769074881.603550:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 393 items remaining. 00000020:00200000:2.0:1769074881.603552:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 392 items remaining. 00000020:00200000:2.0:1769074881.603553:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 391 items remaining. 00000020:00200000:2.0:1769074881.603554:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 390 items remaining. 00000020:00200000:2.0:1769074881.603555:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 389 items remaining. 00000020:00200000:2.0:1769074881.603556:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 388 items remaining. 00000020:00200000:2.0:1769074881.603558:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 387 items remaining. 00000020:00200000:2.0:1769074881.603559:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 386 items remaining. 00000020:00200000:2.0:1769074881.603561:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 385 items remaining. 00000020:00200000:2.0:1769074881.603563:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 384 items remaining. 00000020:00200000:2.0:1769074881.603564:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 383 items remaining. 00000020:00200000:2.0:1769074881.603565:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 382 items remaining. 00000020:00200000:2.0:1769074881.603567:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 381 items remaining. 00000020:00200000:2.0:1769074881.603568:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 380 items remaining. 00000020:00200000:2.0:1769074881.603569:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 379 items remaining. 00000020:00200000:2.0:1769074881.603570:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 378 items remaining. 00000020:00200000:2.0:1769074881.603573:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 377 items remaining. 00000020:00200000:2.0:1769074881.603574:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 376 items remaining. 00000020:00200000:2.0:1769074881.603576:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 375 items remaining. 00000020:00200000:2.0:1769074881.603577:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 374 items remaining. 00000020:00200000:2.0:1769074881.603579:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 373 items remaining. 00000020:00200000:2.0:1769074881.603580:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 372 items remaining. 00000020:00200000:2.0:1769074881.603581:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 371 items remaining. 00000020:00200000:2.0:1769074881.603582:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 370 items remaining. 00000020:00200000:2.0:1769074881.603583:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 369 items remaining. 00000020:00200000:2.0:1769074881.603585:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 368 items remaining. 00000020:00200000:2.0:1769074881.603586:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 367 items remaining. 00000020:00200000:2.0:1769074881.603588:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 366 items remaining. 00000020:00200000:2.0:1769074881.603590:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 365 items remaining. 00000020:00200000:2.0:1769074881.603592:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 364 items remaining. 00000020:00200000:2.0:1769074881.603593:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 363 items remaining. 00000020:00200000:2.0:1769074881.603595:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 362 items remaining. 00000020:00200000:2.0:1769074881.605715:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 361 items remaining. 00000020:00200000:2.0:1769074881.605730:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 360 items remaining. 00000020:00200000:2.0:1769074881.605732:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 359 items remaining. 00000020:00200000:2.0:1769074881.605734:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 358 items remaining. 00000020:00200000:2.0:1769074881.605736:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 357 items remaining. 00000020:00200000:2.0:1769074881.605737:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 356 items remaining. 00000020:00200000:2.0:1769074881.605739:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 355 items remaining. 00000020:00200000:2.0:1769074881.605740:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 354 items remaining. 00000020:00200000:2.0:1769074881.605742:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 353 items remaining. 00000020:00200000:2.0:1769074881.605743:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 352 items remaining. 00000020:00200000:2.0:1769074881.605745:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 351 items remaining. 00000020:00200000:2.0:1769074881.605748:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 350 items remaining. 00000020:00200000:2.0:1769074881.605749:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 349 items remaining. 00000020:00200000:2.0:1769074881.605751:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 348 items remaining. 00000020:00200000:2.0:1769074881.605752:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 347 items remaining. 00000020:00200000:2.0:1769074881.605754:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 346 items remaining. 00000020:00200000:2.0:1769074881.605756:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 345 items remaining. 00000020:00200000:2.0:1769074881.605758:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 344 items remaining. 00000020:00200000:2.0:1769074881.605760:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 343 items remaining. 00000020:00200000:2.0:1769074881.605761:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 342 items remaining. 00000020:00200000:2.0:1769074881.605763:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 341 items remaining. 00000020:00200000:2.0:1769074881.605764:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 340 items remaining. 00000020:00200000:2.0:1769074881.605766:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 339 items remaining. 00000020:00200000:2.0:1769074881.605767:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 338 items remaining. 00000020:00200000:2.0:1769074881.605769:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 337 items remaining. 00000020:00200000:2.0:1769074881.605770:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 336 items remaining. 00000020:00200000:2.0:1769074881.605771:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 335 items remaining. 00000020:00200000:2.0:1769074881.605773:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 334 items remaining. 00000020:00200000:2.0:1769074881.605775:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 333 items remaining. 00000020:00200000:2.0:1769074881.605777:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 332 items remaining. 00000020:00200000:2.0:1769074881.605778:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 331 items remaining. 00000020:00200000:2.0:1769074881.605780:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 330 items remaining. 00000020:00200000:2.0:1769074881.605782:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 329 items remaining. 00000020:00200000:2.0:1769074881.605784:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 328 items remaining. 00000020:00200000:2.0:1769074881.605786:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 327 items remaining. 00000020:00200000:2.0:1769074881.605787:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 326 items remaining. 00000020:00200000:2.0:1769074881.605788:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 325 items remaining. 00000020:00200000:2.0:1769074881.605790:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 324 items remaining. 00000020:00200000:2.0:1769074881.605791:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 323 items remaining. 00000020:00200000:2.0:1769074881.605792:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 322 items remaining. 00000020:00200000:2.0:1769074881.605793:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 321 items remaining. 00000020:00200000:2.0:1769074881.605795:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 320 items remaining. 00000020:00200000:2.0:1769074881.605796:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 319 items remaining. 00000020:00200000:2.0:1769074881.605798:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 318 items remaining. 00000020:00200000:2.0:1769074881.605799:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 317 items remaining. 00000020:00200000:2.0:1769074881.605801:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 316 items remaining. 00000020:00200000:2.0:1769074881.605802:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 315 items remaining. 00000020:00200000:2.0:1769074881.605804:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 314 items remaining. 00000020:00200000:2.0:1769074881.605806:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 313 items remaining. 00000020:00200000:2.0:1769074881.605807:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 312 items remaining. 00000020:00200000:2.0:1769074881.605809:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 311 items remaining. 00000020:00200000:2.0:1769074881.605811:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 310 items remaining. 00000020:00200000:2.0:1769074881.605813:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 309 items remaining. 00000020:00200000:2.0:1769074881.605814:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 308 items remaining. 00000020:00200000:2.0:1769074881.605816:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 307 items remaining. 00000020:00200000:2.0:1769074881.605817:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 306 items remaining. 00000020:00200000:2.0:1769074881.605819:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 305 items remaining. 00000020:00200000:2.0:1769074881.605820:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 304 items remaining. 00000020:00200000:2.0:1769074881.605822:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 303 items remaining. 00000020:00200000:2.0:1769074881.605823:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 302 items remaining. 00000020:00200000:2.0:1769074881.605825:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 301 items remaining. 00000020:00200000:2.0:1769074881.605827:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 300 items remaining. 00000020:00200000:2.0:1769074881.605828:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 299 items remaining. 00000020:00200000:2.0:1769074881.605830:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 298 items remaining. 00000020:00200000:2.0:1769074881.605832:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 297 items remaining. 00000020:00200000:2.0:1769074881.605833:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 296 items remaining. 00000020:00200000:2.0:1769074881.605835:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 295 items remaining. 00000020:00200000:2.0:1769074881.605836:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 294 items remaining. 00000020:00200000:2.0:1769074881.605838:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 293 items remaining. 00000020:00200000:2.0:1769074881.605839:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 292 items remaining. 00000020:00200000:2.0:1769074881.605840:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 291 items remaining. 00000020:00200000:2.0:1769074881.605842:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 290 items remaining. 00000020:00200000:2.0:1769074881.605844:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 289 items remaining. 00000020:00200000:2.0:1769074881.605845:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 288 items remaining. 00000020:00200000:2.0:1769074881.605846:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 287 items remaining. 00000020:00200000:2.0:1769074881.605847:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 286 items remaining. 00000020:00200000:2.0:1769074881.605849:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 285 items remaining. 00000020:00200000:2.0:1769074881.605850:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 284 items remaining. 00000020:00200000:2.0:1769074881.605851:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 283 items remaining. 00000020:00200000:2.0:1769074881.605852:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 282 items remaining. 00000020:00200000:2.0:1769074881.605854:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 281 items remaining. 00000020:00200000:2.0:1769074881.605855:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 280 items remaining. 00000020:00200000:2.0:1769074881.605856:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 279 items remaining. 00000020:00200000:2.0:1769074881.605858:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 278 items remaining. 00000020:00200000:2.0:1769074881.605860:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 277 items remaining. 00000020:00200000:2.0:1769074881.605861:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 276 items remaining. 00000020:00200000:2.0:1769074881.605863:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 275 items remaining. 00000020:00200000:2.0:1769074881.605864:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 274 items remaining. 00000020:00200000:2.0:1769074881.605865:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 273 items remaining. 00000020:00200000:2.0:1769074881.605866:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 272 items remaining. 00000020:00200000:2.0:1769074881.605867:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 271 items remaining. 00000020:00200000:2.0:1769074881.605869:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 270 items remaining. 00000020:00200000:2.0:1769074881.605870:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 269 items remaining. 00000020:00200000:2.0:1769074881.605872:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 268 items remaining. 00000020:00200000:2.0:1769074881.605873:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 267 items remaining. 00000020:00200000:2.0:1769074881.605875:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 266 items remaining. 00000020:00200000:2.0:1769074881.605877:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 265 items remaining. 00000020:00200000:2.0:1769074881.605878:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 264 items remaining. 00000020:00200000:2.0:1769074881.605880:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 263 items remaining. 00000020:00200000:2.0:1769074881.605881:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 262 items remaining. 00000020:00200000:2.0:1769074881.605883:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 261 items remaining. 00000020:00200000:2.0:1769074881.605884:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 260 items remaining. 00000020:00200000:2.0:1769074881.605886:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 259 items remaining. 00000020:00200000:2.0:1769074881.605888:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 258 items remaining. 00000020:00200000:2.0:1769074881.605890:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 257 items remaining. 00000020:00200000:2.0:1769074881.605892:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000020:00200000:2.0:1769074881.605894:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1769074881.605896:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1769074881.605897:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1769074881.605899:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1769074881.605900:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1769074881.605902:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1769074881.605905:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1769074881.605907:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1769074881.605909:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1769074881.605911:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1769074881.605912:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1769074881.605913:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1769074881.605915:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1769074881.605917:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1769074881.605918:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1769074881.605920:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1769074881.605921:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1769074881.605923:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1769074881.605925:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1769074881.605927:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1769074881.605929:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1769074881.605931:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1769074881.605932:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1769074881.605934:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1769074881.605936:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1769074881.605938:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1769074881.605940:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1769074881.605942:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1769074881.605944:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1769074881.605946:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1769074881.605948:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1769074881.605950:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1769074881.605951:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1769074881.605953:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1769074881.605955:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1769074881.605957:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1769074881.605959:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1769074881.605961:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1769074881.605963:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1769074881.605964:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1769074881.605966:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1769074881.605968:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1769074881.605970:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1769074881.605972:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1769074881.605973:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1769074881.605975:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1769074881.605977:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1769074881.605978:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1769074881.605979:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1769074881.605981:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1769074881.605983:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1769074881.605985:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1769074881.605987:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1769074881.605989:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1769074881.605991:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1769074881.605992:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1769074881.605994:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1769074881.605996:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1769074881.605998:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1769074881.606000:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1769074881.606002:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1769074881.606003:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1769074881.606005:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1769074881.606007:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1769074881.606008:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1769074881.606010:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1769074881.606012:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1769074881.606014:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1769074881.606015:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1769074881.606017:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1769074881.606020:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1769074881.606021:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1769074881.606023:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1769074881.606025:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1769074881.606026:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1769074881.606028:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1769074881.606030:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1769074881.606031:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1769074881.606033:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1769074881.606035:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1769074881.606036:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1769074881.606038:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1769074881.606040:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1769074881.606041:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1769074881.606043:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1769074881.606045:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1769074881.606046:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1769074881.606048:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1769074881.606049:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1769074881.606051:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1769074881.606053:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1769074881.606054:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1769074881.606056:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1769074881.606058:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1769074881.606059:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1769074881.606061:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1769074881.606062:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1769074881.606064:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1769074881.606065:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1769074881.606067:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1769074881.606068:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1769074881.606070:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1769074881.606072:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1769074881.606074:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1769074881.606076:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1769074881.606077:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1769074881.606079:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1769074881.606081:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1769074881.606082:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1769074881.606084:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1769074881.607538:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1769074881.607550:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1769074881.607552:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1769074881.607554:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1769074881.607556:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1769074881.607558:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1769074881.607559:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1769074881.607560:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1769074881.607562:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1769074881.607564:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1769074881.607565:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1769074881.607567:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1769074881.607568:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1769074881.607570:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1769074881.607572:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1769074881.607573:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1769074881.607575:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1769074881.607576:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1769074881.607578:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1769074881.607579:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1769074881.607581:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1769074881.607582:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1769074881.607584:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1769074881.607585:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1769074881.607588:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1769074881.607589:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1769074881.607591:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1769074881.607592:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1769074881.607594:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1769074881.607596:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1769074881.607598:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1769074881.607600:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1769074881.607601:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1769074881.607603:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1769074881.607605:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1769074881.607607:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1769074881.607609:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1769074881.607611:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1769074881.607612:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1769074881.607614:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1769074881.607615:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1769074881.607617:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1769074881.607619:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1769074881.607621:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1769074881.607623:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1769074881.607624:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1769074881.607626:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1769074881.607628:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1769074881.607630:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1769074881.607632:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1769074881.607634:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1769074881.607636:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1769074881.607638:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1769074881.607640:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1769074881.607642:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1769074881.607643:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1769074881.607646:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1769074881.607648:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1769074881.607650:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1769074881.607652:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1769074881.607654:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1769074881.607655:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1769074881.607657:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1769074881.607658:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1769074881.607660:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1769074881.607663:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1769074881.607664:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1769074881.607666:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1769074881.607668:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1769074881.607670:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1769074881.607671:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1769074881.607673:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1769074881.607675:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1769074881.607677:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1769074881.607679:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1769074881.607681:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1769074881.607683:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1769074881.607685:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1769074881.607686:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1769074881.607688:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1769074881.607691:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1769074881.607693:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1769074881.607695:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1769074881.607697:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1769074881.607699:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1769074881.607701:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1769074881.607703:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1769074881.607705:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1769074881.607707:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1769074881.607709:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1769074881.607711:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1769074881.607712:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1769074881.607714:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1769074881.607715:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1769074881.607717:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1769074881.607719:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1769074881.607721:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1769074881.607723:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1769074881.607725:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1769074881.607726:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1769074881.607728:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1769074881.607730:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1769074881.607731:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1769074881.607733:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1769074881.607734:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1769074881.607736:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1769074881.607737:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1769074881.607739:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1769074881.607740:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1769074881.607742:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1769074881.607743:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1769074881.607745:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1769074881.607747:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1769074881.607748:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1769074881.607749:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1769074881.607751:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1769074881.607752:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1769074881.607754:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1769074881.607755:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1769074881.607757:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1769074881.607759:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1769074881.607761:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1769074881.607762:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1769074881.607764:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1769074881.607765:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1769074881.607767:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1769074881.607768:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1769074881.607769:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1769074881.607771:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1769074881.607773:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1769074881.607774:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1769074881.607776:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1769074881.607777:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1769074881.607778:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1769074881.607780:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1769074881.607781:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1769074881.607782:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1769074881.607784:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1769074881.607785:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1769074881.607787:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1769074881.607789:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1769074881.607790:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1769074881.607792:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1769074881.607793:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1769074881.607795:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1769074881.607796:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1769074881.608036:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1769074881.608067:0:2409:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b035349f240 00000100:00100000:2.0:1769074881.609453:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617044992:192.168.204.160@tcp:3:lfs.0 00000100:00100000:2.0:1769074881.614301:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617045120:192.168.204.160@tcp:4:lfs.0 00000100:00100000:2.0:1769074881.715068:0:2408:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9b0342ad2800: committing for last_committed 17179874638 gen 2 00000008:00010000:2.0:1769074881.715143:0:2408:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b035349f180 00000020:00200000:2.0:1769074881.715149:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1769074881.715152:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1769074881.715154:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1769074881.715156:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1769074881.715157:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1769074881.715159:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1769074881.715160:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1769074881.715162:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1769074881.715163:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1769074881.715164:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1769074881.715166:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1769074881.715167:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1769074881.715168:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1769074881.715170:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1769074881.715171:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1769074881.715172:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1769074881.715174:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1769074881.715175:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1769074881.715176:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1769074881.715178:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1769074881.715179:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1769074881.715181:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1769074881.715182:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1769074881.715183:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1769074881.715184:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1769074881.715186:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1769074881.715188:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1769074881.715189:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1769074881.715191:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1769074881.715192:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1769074881.715193:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1769074881.715195:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1769074881.715197:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1769074881.715198:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1769074881.715200:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1769074881.715201:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1769074881.715202:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1769074881.715204:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1769074881.715205:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1769074881.715206:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1769074881.715207:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1769074881.715209:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1769074881.715210:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1769074881.715212:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1769074881.715213:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1769074881.715215:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1769074881.715216:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1769074881.715218:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1769074881.715219:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1769074881.715221:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1769074881.715223:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1769074881.715225:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1769074881.715226:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1769074881.715228:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1769074881.715230:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1769074881.715295:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1769074881.715299:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1769074881.715300:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1769074881.715302:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1769074881.715303:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1769074881.715305:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1769074881.715306:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1769074881.715308:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1769074881.715310:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1769074881.715312:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1769074881.715313:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1769074881.715315:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1769074881.715316:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1769074881.715318:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1769074881.715320:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1769074881.715321:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1769074881.715323:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1769074881.715324:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1769074881.715326:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1769074881.715327:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1769074881.715329:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1769074881.715330:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1769074881.715332:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1769074881.715334:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1769074881.715335:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1769074881.715337:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1769074881.715339:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1769074881.715341:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1769074881.715342:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1769074881.715344:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1769074881.715346:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1769074881.715347:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1769074881.715349:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1769074881.715351:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1769074881.715352:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1769074881.715354:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1769074881.715355:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1769074881.715357:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1769074881.715358:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1769074881.715360:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1769074881.715363:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1769074881.715364:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1769074881.715366:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1769074881.715368:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1769074881.715369:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1769074881.715371:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1769074881.715372:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1769074881.715373:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1769074881.715375:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1769074881.715376:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1769074881.715378:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1769074881.715380:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1769074881.715382:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1769074881.715383:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1769074881.715385:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1769074881.715386:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1769074881.715388:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1769074881.715389:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1769074881.715391:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1769074881.715392:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1769074881.715393:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1769074881.715395:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1769074881.715396:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1769074881.715398:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1769074881.715399:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1769074881.715400:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1769074881.715402:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1769074881.715404:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1769074881.715405:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1769074881.715407:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1769074881.715408:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1769074881.715410:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1769074881.715412:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1769074881.715413:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1769074881.715415:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1769074881.715416:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1769074881.715417:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1769074881.715419:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1769074881.715420:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1769074881.715421:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1769074881.715423:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1769074881.715424:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1769074881.715426:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1769074881.715427:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1769074881.715429:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1769074881.715431:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1769074881.715432:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1769074881.715434:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1769074881.715435:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1769074881.715437:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1769074881.715439:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1769074881.715440:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1769074881.715442:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1769074881.715443:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1769074881.715445:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1769074881.715446:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1769074881.715448:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1769074881.715449:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1769074881.715451:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1769074881.715452:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1769074881.715453:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1769074881.715455:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1769074881.715456:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1769074881.715458:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1769074881.715460:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1769074881.715461:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1769074881.715463:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1769074881.715464:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1769074881.715466:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1769074881.715467:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1769074881.715468:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1769074881.715470:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1769074881.715471:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1769074881.715473:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1769074881.715474:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1769074881.715476:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1769074881.715477:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1769074881.715479:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1769074881.715480:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1769074881.715481:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1769074881.715482:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1769074881.715484:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1769074881.715486:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1769074881.715488:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1769074881.715489:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1769074881.715491:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1769074881.715492:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1769074881.715494:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1769074881.715496:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1769074881.715497:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1769074881.715499:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1769074881.715500:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1769074881.715502:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1769074881.715504:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1769074881.715505:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1769074881.715506:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1769074881.715508:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1769074881.715509:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1769074881.715511:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1769074881.715513:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1769074881.715514:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1769074881.715515:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1769074881.715517:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1769074881.715518:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1769074881.715520:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1769074881.715521:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1769074881.715523:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1769074881.715524:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1769074881.715526:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1769074881.715527:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1769074881.715529:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1769074881.715531:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1769074881.715533:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1769074881.715534:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1769074881.715536:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1769074881.715537:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1769074881.715539:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1769074881.715541:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1769074881.715542:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1769074881.715544:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1769074881.715546:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1769074881.715547:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1769074881.715549:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1769074881.715550:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1769074881.715552:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1769074881.715554:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1769074881.715555:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1769074881.715556:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1769074881.715558:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1769074881.715560:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1769074881.715561:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1769074881.715563:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1769074881.715564:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1769074881.715566:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1769074881.715567:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1769074881.715569:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1769074881.715570:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1769074881.715572:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1769074881.715573:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1769074881.715575:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1769074881.715576:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1769074881.715578:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1769074881.715579:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1769074881.715581:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1769074881.715582:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1769074881.715584:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1769074881.715585:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1769074881.715587:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1769074881.715588:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1769074881.715589:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1769074881.715590:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1769074881.715592:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1769074881.715593:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1769074881.715595:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1769074881.715596:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1769074881.715598:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1769074881.715600:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1769074881.715601:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1769074881.715602:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1769074881.715604:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1769074881.715606:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1769074881.715797:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1769074881.715822:0:2408:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b035349f180 00000100:00100000:2.0:1769074881.715856:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617045120:192.168.204.160@tcp:4:lfs.0 00000100:00100000:2.0:1769074881.718317:0:2408:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074881.718342:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617045248:192.168.204.160@tcp:4:lfs.0 00000100:00100000:2.0:1769074881.783222:0:2408:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9b0342ad2800: committing for last_committed 17179874639 gen 2 00000008:00010000:2.0:1769074881.783316:0:2408:0:(osc_cache.c:861:osc_extent_finish()) ### no dlm lock: extent: ffff9b035349fe40 00000020:00200000:2.0:1769074881.783323:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1769074881.783326:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1769074881.783327:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1769074881.783329:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1769074881.783331:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1769074881.783333:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1769074881.783335:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1769074881.783336:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1769074881.783338:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1769074881.783340:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1769074881.783342:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1769074881.783343:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1769074881.783345:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1769074881.783347:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1769074881.783348:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1769074881.783350:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1769074881.783351:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1769074881.783353:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1769074881.783354:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1769074881.783356:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1769074881.783357:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1769074881.783359:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1769074881.783361:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1769074881.783364:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1769074881.783365:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1769074881.783367:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1769074881.783368:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1769074881.783370:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1769074881.783371:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1769074881.783373:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1769074881.783374:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1769074881.783375:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1769074881.783377:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1769074881.783378:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1769074881.783380:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1769074881.783381:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1769074881.783383:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1769074881.783385:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1769074881.783386:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1769074881.783388:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1769074881.783390:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1769074881.783392:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1769074881.783394:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1769074881.783396:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1769074881.783397:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1769074881.783399:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1769074881.783400:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1769074881.783402:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1769074881.783403:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1769074881.783404:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1769074881.783406:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1769074881.783407:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1769074881.783409:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1769074881.783411:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1769074881.783413:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1769074881.783415:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1769074881.783416:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1769074881.783418:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1769074881.783420:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1769074881.783421:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1769074881.783422:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1769074881.783424:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1769074881.783425:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1769074881.783427:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1769074881.783428:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1769074881.783430:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1769074881.783432:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1769074881.783434:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1769074881.783435:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1769074881.783437:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1769074881.783439:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1769074881.783440:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1769074881.783442:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1769074881.783444:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1769074881.783445:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1769074881.783447:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1769074881.783449:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1769074881.783450:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1769074881.783452:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1769074881.783453:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1769074881.783455:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1769074881.783457:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1769074881.783458:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1769074881.783460:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1769074881.783462:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1769074881.783463:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1769074881.783465:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1769074881.783466:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1769074881.783467:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1769074881.783469:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1769074881.783471:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1769074881.783472:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1769074881.783473:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1769074881.783474:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1769074881.783476:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1769074881.783477:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1769074881.783478:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1769074881.783480:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1769074881.783482:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1769074881.783483:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1769074881.783485:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1769074881.783486:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1769074881.783488:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1769074881.783489:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1769074881.783491:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1769074881.783492:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1769074881.783493:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1769074881.783495:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1769074881.783496:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1769074881.783497:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1769074881.783499:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1769074881.783500:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1769074881.783502:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1769074881.783503:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1769074881.783504:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1769074881.783506:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1769074881.783507:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1769074881.783509:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1769074881.783511:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1769074881.783512:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1769074881.783514:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1769074881.783516:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1769074881.783517:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1769074881.783519:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1769074881.783520:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1769074881.783522:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1769074881.783523:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1769074881.783525:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1769074881.783526:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1769074881.783528:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1769074881.783529:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1769074881.783530:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1769074881.783531:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1769074881.783533:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1769074881.783534:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1769074881.783536:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1769074881.783537:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1769074881.783539:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1769074881.783540:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1769074881.783542:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1769074881.783543:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1769074881.783544:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1769074881.783546:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1769074881.783547:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1769074881.783549:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1769074881.783550:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1769074881.783552:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1769074881.783553:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1769074881.783555:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1769074881.783556:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1769074881.783558:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1769074881.783560:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1769074881.783561:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1769074881.783563:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1769074881.783564:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1769074881.783566:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1769074881.783567:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1769074881.783569:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1769074881.783570:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1769074881.783572:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1769074881.783573:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1769074881.783574:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1769074881.783576:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1769074881.783577:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1769074881.783579:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1769074881.783580:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1769074881.783582:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1769074881.783583:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1769074881.783584:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1769074881.783586:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1769074881.783588:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1769074881.783589:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1769074881.783591:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1769074881.783592:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1769074881.783593:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1769074881.783595:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1769074881.783596:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1769074881.783597:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1769074881.783599:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1769074881.783600:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1769074881.783601:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1769074881.783603:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1769074881.783605:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1769074881.783606:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1769074881.783608:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1769074881.783609:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1769074881.783611:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1769074881.783612:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1769074881.783614:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1769074881.783615:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1769074881.783617:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1769074881.783619:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1769074881.783621:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1769074881.783622:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1769074881.783624:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1769074881.783625:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1769074881.783627:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1769074881.783628:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1769074881.783629:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1769074881.783631:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1769074881.783632:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1769074881.783633:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1769074881.783635:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1769074881.783636:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1769074881.783637:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1769074881.783638:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1769074881.783640:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1769074881.783641:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1769074881.783642:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1769074881.783644:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1769074881.783645:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1769074881.783646:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1769074881.783648:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1769074881.783649:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1769074881.783651:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1769074881.783652:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1769074881.783654:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1769074881.783655:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1769074881.783657:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1769074881.783658:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1769074881.783659:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1769074881.783661:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1769074881.783662:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1769074881.783663:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1769074881.783665:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1769074881.783666:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1769074881.783668:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1769074881.783669:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1769074881.783671:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1769074881.783672:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1769074881.783673:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1769074881.783675:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1769074881.783676:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1769074881.783677:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1769074881.783679:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1769074881.783680:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1769074881.783681:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1769074881.783682:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1769074881.783684:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1769074881.783685:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1769074881.783686:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1769074881.783687:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1769074881.783688:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1769074881.783689:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1769074881.783691:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1769074881.783692:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1769074881.783695:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1769074881.783697:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1769074881.783698:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1769074881.783700:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1769074881.783701:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1769074881.783703:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1769074881.783704:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1769074881.783706:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1769074881.783707:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1769074881.783709:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1769074881.783794:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1769074881.783817:0:2408:0:(osc_cache.c:483:__osc_extent_remove()) ### no dlm lock: extent: ffff9b035349fe40 00000100:00100000:2.0:1769074881.783842:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617045248:192.168.204.160@tcp:4:lfs.0 00000080:00200080:0.0:1769074881.839091:0:67463:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248) cmd=400866b1 arg=2 00000080:00200008:0.0:1769074881.839111:0:67463:0:(file.c:6252:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:0.0:1769074881.839120:0:67463:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248),name=f204c.sanity-flr 00000002:00010000:0.0:1769074881.839147:0:67463:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20a:0x0]) in obj [0x200000405:0x20a:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074881.839168:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074881.839183:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074881.839190:0:67463:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000405:0x20a:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074881.839197:0:67463:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) 00000080:00010000:0.0:1769074881.839201:0:67463:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c919 00000080:00010000:0.0:1769074881.839208:0:67463:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c919 from it ffffb43641cb7cd8 00010000:00010000:0.0:1769074881.839211:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074881.839220:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:0.0:1769074881.839260:0:67463:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 0/1, layout version 8 restore needed 0 00020000:00200000:0.0:1769074881.839282:0:67463:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:0.0:1769074881.839285:0:67463:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:0.0:1769074881.839289:0:67463:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200000405:0x20a:0x0] 00000008:00010000:0.0:1769074881.839308:0:67463:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:0.0:1769074881.839311:0:67463:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b03568dfcb0, osc lock ffff9b0345a6d430, flags 0x400000400001002 00010000:00010000:0.0:1769074881.839328:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.839338:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 2/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074881.839350:0:67463:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:0.0:1769074881.839354:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.839362:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 4/0,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000020:00200000:0.0:1769074881.839371:0:67463:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1769074881.839380:0:67463:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x20a:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1769074881.839385:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03513e76c0 (ffff9b0342ca8ab8/0) at cl_lock_release():245 00000020:00010000:0.0:1769074881.839387:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03513e76c0 (ffff9b0342ca8ab8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074881.839390:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03568dfcb0 (ffff9b0342ca85a8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074881.839394:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03513e76c0 (ffff9b0342ca8ab8/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074881.839396:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03568dfcb0 (ffff9b0342ca8ab8/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074881.839406:0:67463:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 0/1, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1769074881.839432:0:67463:0:(llite_lib.c:2314:ll_setattr_raw()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248) current size 2097152, valid attrs 2068, mode 0, uid -1802657998, gid -1, new size 2097152, atime 1769074881.000000000, mtime 1769074881.000000000, ctime 1769074881.000000000 00010000:00010000:0.0:1769074881.839456:0:67463:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074881.839471:0:67463:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074881.839511:0:67463:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074881.839518:0:67463:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074881.839520:0:67463:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0342af5600/0xb7d6bf619879c919 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55b55 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000020:00100000:0.0:1769074881.839538:0:67463:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074881.839549:0:67463:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a38e00 x1855007617045504/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074881.839572:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617045504:192.168.204.160@tcp:36:lfs.0 00000100:00100000:0.0:1769074881.839666:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ab40 going to sleep for 50 seconds 00000100:00100000:1.0:1769074881.846689:0:67463:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617045504:192.168.204.160@tcp:36:lfs.0 00000080:00200000:1.0:1769074881.846742:0:67463:0:(file.c:1759:ll_io_set_mirror()) f204c.sanity-flr: desiginated mirror: 2 00000080:00200000:1.0:1769074881.846751:0:67463:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074881.846777:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x11 00000080:00200000:1.0:1769074881.846829:0:67463:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1769074881.846862:0:67463:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074881.846866:0:67463:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0372e6d6f0, osc lock ffff9b0345a09560, flags 0x0 00000008:00010000:1.0:1769074881.846871:0:67463:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074881.846872:0:67463:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0372e6d730, osc lock ffff9b0345a08720, flags 0x0 00010000:00010000:1.0:1769074881.846942:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.846955:0:67463:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.846965:0:67463:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074881.847006:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617045632:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074881.847098:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372e6dd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074881.860342:0:67463:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617045632:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074881.860372:0:67463:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.860391:0:67463:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.860405:0:67463:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.860419:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074881.860432:0:67463:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 4/0,2 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.860445:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.860452:0:67463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.860524:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.860532:0:67463:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [1048576->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.860540:0:67463:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 3/0,1 mode: --/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [1048576->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074881.860566:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617045760:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074881.860665:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372e6dd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074881.870306:0:67463:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617045760:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074881.870335:0:67463:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 4/0,1 mode: --/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.870351:0:67463:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.870363:0:67463:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.870376:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000008:00010000:1.0:1769074881.870388:0:67463:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 4/0,2 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.870401:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.870409:0:67463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074881.870572:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617045888:192.168.204.160@tcp:10:lfs.0 00000100:00100000:1.0:1769074881.874576:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617045888:192.168.204.160@tcp:10:lfs.0 00000080:00200000:0.0:1769074881.874779:0:67463:0:(file.c:1686:ll_merge_attr_nolock()) [0x200000405:0x20a:0x0] updating i_size 2097152 i_blocks 4096 00000020:00010000:0.0:1769074881.874787:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0359283e48 (ffff9b035121f128/0) at cl_lock_release():245 00000020:00010000:0.0:1769074881.874790:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359283e48 (ffff9b035121f128/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074881.874796:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0372e6d6f0 (ffff9b035121f878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074881.874805:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.874818:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074881.874835:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0372e6d730 (ffff9b035121f2d8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074881.874839:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.874847:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074881.874855:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359283e48 (ffff9b035121f128/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074881.874858:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0372e6d6f0 (ffff9b035121f128/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074881.874863:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0372e6d730 (ffff9b035121f128/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074881.874882:0:67463:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:0.0:1769074881.874895:0:67463:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248) current size 2097152, valid attrs 2168, mode 0, uid -1802657998, gid -1, new size 2097152, atime 1769074881.000000000, mtime 1769074881.000000000, ctime 1769074881.000000000, rc 0 00000080:00200080:0.0:1769074881.874918:0:67463:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248) cmd=400866b1 arg=0 00000080:00200008:0.0:1769074881.874966:0:67463:0:(llite_lib.c:2314:ll_setattr_raw()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248) current size 2097152, valid attrs 101f0, mode b436, uid 1618051720, gid -25853, new size 18446633037359624776, atime 1769074855.000000000, mtime 1769074867.000000000, ctime 1769074881.000000000 00000020:00100000:0.0:1769074881.875034:0:67463:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074881.875044:0:67463:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a39500 x1855007617046144/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074881.875069:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617046144:192.168.204.160@tcp:36:lfs.0 00000100:00100000:0.0:1769074881.875451:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ac00 going to sleep for 50 seconds 00000100:00100000:1.0:1769074881.884650:0:67463:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617046144:192.168.204.160@tcp:36:lfs.0 00000080:00200000:1.0:1769074881.884729:0:67463:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074881.884744:0:67463:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074881.884747:0:67463:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1769074881.884755:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1769074881.884780:0:67463:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x0 00000080:00200000:1.0:1769074881.884783:0:67463:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:1.0:1769074881.884798:0:67463:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1769074881.884802:0:67463:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0360af63b0, osc lock ffff9b0345a08390, flags 0x0 00010000:00010000:1.0:1769074881.884819:0:67463:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b034921d000 type 11 mode 2 res 10737419265/713 (0 10485759) 00010000:00010000:1.0:1769074881.884863:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.884874:0:67463:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074881.884883:0:67463:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000100:00100000:1.0:1769074881.884902:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617046272:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074881.885561:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372e6dd80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074881.898312:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074881.901207:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625254784 00000100:00100000:0.0:1769074881.901224:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625254784 00000100:00100000:0.0:1769074881.901805:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8951 00000100:00100000:0.0:1769074881.901819:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ceb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:28955:x1855007625254784:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074881.901836:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.901935:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074881.901942:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ceb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:28955:x1855007625254784:12345-192.168.204.160@tcp:104: Request processed in 127us (3641us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074881.901949:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8951 00010000:00010000:2.0:1769074881.902022:0:39339:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074881.902030:0:39339:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.902039:0:39339:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035005b400 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:2.0:1769074881.902045:0:39339:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800428400000000 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074881.902066:0:39339:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074881.902083:0:39339:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769074881.902127:0:39339:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 2/0,0 mode: --/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.902136:0:39339:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074881.902171:0:39339:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 1/0,0 mode: --/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.902184:0:39339:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879c920 lrc: 0/0,0 mode: --/PR res: [0x280000401:0x2c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55b5c expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.902200:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074881.902201:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074881.902203:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074881.902211:0:2409:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074881.902225:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617046400:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074881.929733:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:2.0:1769074881.929948:0:67463:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617046272:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074881.929973:0:67463:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x10000000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.929986:0:67463:0:(ldlm_request.c:274:ldlm_completion_ast()) ### client-side enqueue returned a blocked locksleeping ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.929995:0:67463:0:(ldlm_request.c:303:ldlm_completion_ast()) ### waiting indefinitely because of NO_TIMEOUT ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 4/0,1 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000100:00100000:0.0:1769074881.930584:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625254912 00000100:00100000:0.0:1769074881.930594:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625254912 00000100:00100000:0.0:1769074881.930667:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8952 00000100:00100000:0.0:1769074881.930677:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6500:x1855007625254912:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074881.930687:0:55997:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.930697:0:55997:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.930706:0:55997:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 5/0,1 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.930714:0:55997:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10080000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.930722:0:55997:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10080000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.930736:0:55997:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x80000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074881.930811:0:55997:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b035919ca00) 00000100:00100000:0.0:1769074881.930868:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce8700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:6500:x1855007625254912:12345-192.168.204.160@tcp:105: Request processed in 193us (1146us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074881.930876:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8952 00010000:00010000:2.0:1769074881.937632:0:67463:0:(ldlm_request.c:186:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.937651:0:67463:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.937668:0:67463:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 5/0,2 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074881.937682:0:67463:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 2097152, setting rss=2097152; kms=2097152, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 4/0,2 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10020000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.937695:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 5/0,2 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.937701:0:67463:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074881.943302:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0359283c68 (ffff9b035121f368/0) at cl_lock_release():245 00000020:00010000:2.0:1769074881.943313:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359283c68 (ffff9b035121f368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074881.943319:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0360af63b0 (ffff9b035121f908/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074881.943332:0:67463:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074881.943346:0:67463:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074881.943363:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359283c68 (ffff9b035121f368/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074881.943366:0:67463:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0360af63b0 (ffff9b035121f368/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074881.943389:0:67463:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 0/0, layout version 8 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074881.943404:0:67463:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248) current size 2097152, valid attrs 101f0, mode b436, uid 1618051720, gid -25853, new size 18446633037359624776, atime 1769074855.000000000, mtime 1769074867.000000000, ctime 1769074881.000000000, rc 0 00000080:00200080:2.0:1769074881.943427:0:67463:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248) cmd=c00c66f3 arg=b802a0 00010000:00010000:2.0:1769074881.943492:0:67463:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c90b lrc: 2/0,0 mode: PW/PW res: [0x200000405:0x20a:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7fe2323adfd55b2b expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074881.943509:0:67463:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879c90b lrc: 0/0,0 mode: --/PW res: [0x200000405:0x20a:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7fe2323adfd55b2b expref: -99 pid: 67463 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074881.943532:0:67463:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b03482c5f80 x1855007617043584/t8589944131(8589944131) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 968/1056 e 0 to 0 dl 1769074936 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074881.943543:0:67463:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074881.943549:0:67463:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7b80 x1855007617046784/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 504/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074881.943572:0:67463:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617046784:192.168.204.160@tcp:35:lfs.0 00000100:00100000:2.0:1769074881.943665:0:67463:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 11 seconds 00000100:00100000:1.0:1769074881.946132:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074881.947497:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625255040 00000100:00100000:0.0:1769074881.947513:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625255040 00000100:00100000:0.0:1769074881.947583:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8953 00000100:00100000:0.0:1769074881.947593:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:15947:x1855007625255040:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074881.947610:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074881.947694:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074881.947701:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:15947:x1855007625255040:12345-192.168.204.160@tcp:104: Request processed in 109us (1577us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074881.947708:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8953 00010000:00010000:2.0:1769074881.948053:0:60112:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074881.948061:0:60112:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.948071:0:60112:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860a00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074881.948076:0:60112:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.948085:0:60112:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.948093:0:60112:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 3/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074881.948118:0:60112:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x20a:0x0]: losing layout lock 00010000:00010000:2.0:1769074881.948158:0:60112:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 2/0,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.948176:0:60112:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074881.948195:0:60112:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 1/0,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.948201:0:60112:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c912 lrc: 0/0,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd55b47 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074881.948218:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074881.948219:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074881.948221:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074881.959361:0:67463:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944136 gen 1 00000100:00100000:2.0:1769074881.959392:0:67463:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67463:1855007617046784:192.168.204.160@tcp:35:lfs.0 00010000:00010000:2.0:1769074882.011809:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:0.0:1769074882.052692:0:67466:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074882.052704:0:67466:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074882.052730:0:67466:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074882.052748:0:67466:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074882.052764:0:67466:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074882.052777:0:67466:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074882.052784:0:67466:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074882.052788:0:67466:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:0.0:1769074882.052796:0:67466:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641cd7cb8 00010000:00010000:0.0:1769074882.052800:0:67466:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074882.052808:0:67466:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074882.052935:0:67466:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074882.052945:0:67466:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074882.052952:0:67466:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074882.052960:0:67466:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074882.052965:0:67466:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074882.052969:0:67466:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074882.052971:0:67466:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:0.0:1769074882.052974:0:67466:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641cd7bb0 00010000:00010000:0.0:1769074882.052977:0:67466:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074882.052983:0:67466:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074882.052994:0:67466:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074882.053001:0:67466:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:0.0:1769074882.053018:0:67466:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7fffde5105dc 00000080:00200008:0.0:1769074882.053031:0:67466:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:1.0:1769074882.060317:0:67469:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074882.060351:0:67469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074882.060369:0:67469:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.060385:0:67469:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.060393:0:67469:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074882.060401:0:67469:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074882.060405:0:67469:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:1.0:1769074882.060412:0:67469:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb4364155bbf8 00010000:00010000:1.0:1769074882.060417:0:67469:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.060425:0:67469:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.060440:0:67469:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074882.060447:0:67469:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:1.0:1769074882.060453:0:67469:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:1.0:1769074882.060465:0:67469:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), file ffff9b034583bc80, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1769074882.060471:0:67469:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=open 00000002:00010000:1.0:1769074882.060487:0:67469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x9:0x0], intent: open flags 0304001 00010000:00010000:1.0:1769074882.060544:0:67469:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67469 initiator: MDT0 00010000:00010000:1.0:1769074882.060553:0:67469:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67469 initiator: MDT0 00000020:00100000:1.0:1769074882.060564:0:67469:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074882.060578:0:67469:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342cead80 x1855007617047936/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074882.060586:0:67469:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67469 initiator: MDT0 00000100:00100000:1.0:1769074882.060620:0:67469:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67469:1855007617047936:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074882.060943:0:67469:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372e6d240 going to sleep for 50 seconds 00000100:00100000:1.0:1769074882.070653:0:67469:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67469:1855007617047936:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074882.070679:0:67469:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67469 initiator: MDT0 00010000:00010000:1.0:1769074882.070692:0:67469:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67469 initiator: MDT0 00010000:00010000:1.0:1769074882.070699:0:67469:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67469 initiator: MDT0 00010000:00010000:1.0:1769074882.070706:0:67469:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67469 initiator: MDT0 00010000:00010000:1.0:1769074882.070713:0:67469:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67469 initiator: MDT0 00010000:00010000:1.0:1769074882.070719:0:67469:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919d800 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:1.0:1769074882.070724:0:67469:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67469 initiator: MDT0 00010000:00010000:1.0:1769074882.070730:0:67469:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67469 initiator: MDT0 00010000:00010000:1.0:1769074882.070738:0:67469:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 67469 initiator: MDT0 00010000:00010000:1.0:1769074882.070749:0:67469:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074882.070755:0:67469:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67469 initiator: MDT0 00010000:00010000:1.0:1769074882.070760:0:67469:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919d800/0xb7d6bf619879c94a lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67469 initiator: MDT0 00000002:00100000:1.0:1769074882.070788:0:67469:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0342cead80 x1855007617047936/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074937 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074882.070803:0:67469:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1769074882.071062:0:67469:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074882.071083:0:67469:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074882.071097:0:67469:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.071109:0:67469:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.071116:0:67469:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074882.071121:0:67469:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074882.071124:0:67469:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:1.0:1769074882.071130:0:67469:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb4364155bbf8 00010000:00010000:1.0:1769074882.071133:0:67469:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.071140:0:67469:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.071155:0:67469:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074882.071163:0:67469:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:1.0:1769074882.071171:0:67469:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 44 00000080:00200008:1.0:1769074882.071180:0:67469:0:(file.c:1001:ll_file_open()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00000080:00200080:1.0:1769074882.071229:0:67469:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908) cmd=c1406916 arg=18bfa10 00000100:00100000:1.0:1769074882.071335:0:67469:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67469:1855007617048064:192.168.204.160@tcp:34:lfs.0 00000100:00100000:1.0:1769074882.071505:0:67469:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372e6d240 going to sleep for 50 seconds 00000080:00200008:1.0:1769074882.123018:0:67473:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074882.123029:0:67473:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074882.123053:0:67473:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074882.123071:0:67473:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.123086:0:67473:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.123095:0:67473:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074882.123103:0:67473:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074882.123106:0:67473:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:1.0:1769074882.123114:0:67473:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641cd7cb8 00010000:00010000:1.0:1769074882.123117:0:67473:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.123125:0:67473:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.123747:0:67473:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074882.123760:0:67473:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074882.123769:0:67473:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.123782:0:67473:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.123789:0:67473:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074882.123793:0:67473:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074882.123796:0:67473:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:1.0:1769074882.123800:0:67473:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641cd7bb0 00010000:00010000:1.0:1769074882.123804:0:67473:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.123810:0:67473:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.123819:0:67473:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074882.123828:0:67473:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074882.123847:0:67473:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffcffcfe25c 00000080:00200008:1.0:1769074882.123860:0:67473:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:2.0:1769074882.125597:0:67475:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074882.125627:0:67475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074882.125645:0:67475:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074882.125659:0:67475:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074882.125667:0:67475:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074882.125675:0:67475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074882.125678:0:67475:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074882.125685:0:67475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641dbbbf8 00010000:00010000:2.0:1769074882.125688:0:67475:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074882.125701:0:67475:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074882.125714:0:67475:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074882.125721:0:67475:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:2.0:1769074882.125727:0:67475:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:2.0:1769074882.125737:0:67475:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), file ffff9b03591d6f00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074882.125744:0:67475:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=open 00000002:00010000:2.0:1769074882.125760:0:67475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x9:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074882.125817:0:67475:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67475 initiator: MDT0 00010000:00010000:2.0:1769074882.125825:0:67475:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67475 initiator: MDT0 00000020:00100000:2.0:1769074882.125837:0:67475:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074882.125849:0:67475:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7800 x1855007617048320/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074882.125857:0:67475:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67475 initiator: MDT0 00000100:00100000:2.0:1769074882.125878:0:67475:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67475:1855007617048320:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074882.130006:0:67475:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074882.141105:0:67475:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67475:1855007617048320:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074882.141129:0:67475:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67475 initiator: MDT0 00010000:00010000:2.0:1769074882.141146:0:67475:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67475 initiator: MDT0 00010000:00010000:2.0:1769074882.141152:0:67475:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67475 initiator: MDT0 00010000:00010000:2.0:1769074882.141159:0:67475:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67475 initiator: MDT0 00010000:00010000:2.0:1769074882.141166:0:67475:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67475 initiator: MDT0 00010000:00010000:2.0:1769074882.141171:0:67475:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860a00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074882.141176:0:67475:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67475 initiator: MDT0 00010000:00010000:2.0:1769074882.141183:0:67475:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67475 initiator: MDT0 00010000:00010000:2.0:1769074882.141189:0:67475:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 67475 initiator: MDT0 00010000:00010000:2.0:1769074882.141196:0:67475:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074882.141201:0:67475:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67475 initiator: MDT0 00010000:00010000:2.0:1769074882.141207:0:67475:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c951 lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67475 initiator: MDT0 00000002:00100000:2.0:1769074882.141278:0:67475:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b03482c7800 x1855007617048320/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074937 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074882.141293:0:67475:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074882.141760:0:67475:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074882.141785:0:67475:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074882.141802:0:67475:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074882.141812:0:67475:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074882.141819:0:67475:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074882.141824:0:67475:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074882.141827:0:67475:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074882.141832:0:67475:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641dbbbf8 00010000:00010000:2.0:1769074882.141834:0:67475:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074882.141842:0:67475:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074882.141855:0:67475:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074882.141862:0:67475:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:2.0:1769074882.141869:0:67475:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 44 00000080:00200008:2.0:1769074882.141877:0:67475:0:(file.c:1001:ll_file_open()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00000080:00200080:2.0:1769074882.141926:0:67475:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908) cmd=c1406916 arg=258ea10 00000100:00100000:2.0:1769074882.141989:0:67475:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67475:1855007617048448:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074882.142069:0:67475:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074882.153842:0:67475:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67475:1855007617048448:192.168.204.160@tcp:34:lfs.0 00000080:00200008:2.0:1769074882.153912:0:67475:0:(file.c:424:ll_file_release()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00010000:00010000:2.0:1769074882.154174:0:67475:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b800/0xb7d6bf619879c8e1 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x209:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a75 expref: -99 pid: 67448 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074882.188619:0:67480:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074882.188696:0:67480:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074882.188720:0:67480:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074882.188738:0:67480:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.188753:0:67480:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.188760:0:67480:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074882.188768:0:67480:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074882.188773:0:67480:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:1.0:1769074882.188780:0:67480:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641ddbcb8 00010000:00010000:1.0:1769074882.188784:0:67480:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.188792:0:67480:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.188931:0:67480:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074882.189033:0:67480:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074882.189041:0:67480:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.189049:0:67480:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.189055:0:67480:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074882.189059:0:67480:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074882.189062:0:67480:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:1.0:1769074882.189065:0:67480:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641ddbbb0 00010000:00010000:1.0:1769074882.189069:0:67480:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074882.189075:0:67480:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074882.189169:0:67480:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074882.189179:0:67480:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074882.189198:0:67480:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffc7e66fa0c 00000080:00200008:1.0:1769074882.189212:0:67480:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:2.0:1769074882.192541:0:67484:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074882.192573:0:67484:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074882.192589:0:67484:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074882.192603:0:67484:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074882.192610:0:67484:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074882.192617:0:67484:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074882.192620:0:67484:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074882.192626:0:67484:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641e3bbf8 00010000:00010000:2.0:1769074882.192630:0:67484:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074882.192637:0:67484:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074882.192652:0:67484:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074882.192658:0:67484:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:2.0:1769074882.192664:0:67484:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:2.0:1769074882.192675:0:67484:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), file ffff9b03591d6000, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074882.192681:0:67484:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=open 00000002:00010000:2.0:1769074882.192705:0:67484:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x9:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074882.192759:0:67484:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67484 initiator: MDT0 00010000:00010000:2.0:1769074882.192770:0:67484:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67484 initiator: MDT0 00000020:00100000:2.0:1769074882.192782:0:67484:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074882.192794:0:67484:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7800 x1855007617048704/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074882.192804:0:67484:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67484 initiator: MDT0 00000100:00100000:2.0:1769074882.192826:0:67484:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67484:1855007617048704:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074882.201169:0:67484:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074882.214859:0:67484:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67484:1855007617048704:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074882.214885:0:67484:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67484 initiator: MDT0 00010000:00010000:2.0:1769074882.214899:0:67484:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67484 initiator: MDT0 00010000:00010000:2.0:1769074882.214905:0:67484:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67484 initiator: MDT0 00010000:00010000:2.0:1769074882.214912:0:67484:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67484 initiator: MDT0 00010000:00010000:2.0:1769074882.214918:0:67484:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67484 initiator: MDT0 00010000:00010000:2.0:1769074882.215167:0:67484:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860a00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074882.215173:0:67484:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67484 initiator: MDT0 00010000:00010000:2.0:1769074882.215181:0:67484:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67484 initiator: MDT0 00010000:00010000:2.0:1769074882.215190:0:67484:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 67484 initiator: MDT0 00010000:00010000:2.0:1769074882.215197:0:67484:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074882.215204:0:67484:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67484 initiator: MDT0 00010000:00010000:2.0:1769074882.215210:0:67484:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879c958 lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67484 initiator: MDT0 00000002:00100000:2.0:1769074882.215291:0:67484:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b03482c7800 x1855007617048704/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074937 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074882.215305:0:67484:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074882.215544:0:67484:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074882.215565:0:67484:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074882.215578:0:67484:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074882.215590:0:67484:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074882.215597:0:67484:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074882.215602:0:67484:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074882.215606:0:67484:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074882.215611:0:67484:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641e3bbf8 00010000:00010000:2.0:1769074882.215615:0:67484:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074882.215627:0:67484:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074882.215640:0:67484:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074882.215647:0:67484:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:2.0:1769074882.215653:0:67484:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 44 00000080:00200008:2.0:1769074882.215662:0:67484:0:(file.c:1001:ll_file_open()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00000080:00200080:2.0:1769074882.215706:0:67484:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908) cmd=c1406916 arg=bb7a10 00000100:00100000:2.0:1769074882.215765:0:67484:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67484:1855007617048832:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074882.215859:0:67484:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b55b40 going to sleep for 50 seconds 00000100:00100000:2.0:1769074882.222454:0:67484:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67484:1855007617048832:192.168.204.160@tcp:34:lfs.0 00000080:00200008:2.0:1769074882.222625:0:67484:0:(file.c:424:ll_file_release()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00010000:00010000:2.0:1769074882.222657:0:67484:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b800/0xb7d6bf619879c8e1 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x209:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a75 expref: -99 pid: 67448 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074882.240567:0:67486:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074882.240599:0:67486:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074882.240713:0:67486:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074882.240731:0:67486:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074882.240740:0:67486:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074882.240748:0:67486:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074882.240752:0:67486:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:0.0:1769074882.240760:0:67486:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641da7bf8 00010000:00010000:0.0:1769074882.240764:0:67486:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074882.240773:0:67486:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074882.240790:0:67486:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074882.240798:0:67486:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=27 00000080:00200000:0.0:1769074882.240804:0:67486:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:0.0:1769074882.240815:0:67486:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), file ffff9b034583b080, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1769074882.240823:0:67486:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=open 00000002:00010000:0.0:1769074882.240840:0:67486:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x9:0x0], intent: open flags 0304001 00010000:00010000:0.0:1769074882.240898:0:67486:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67486 initiator: MDT0 00010000:00010000:0.0:1769074882.240906:0:67486:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67486 initiator: MDT0 00000020:00100000:0.0:1769074882.240919:0:67486:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074882.240932:0:67486:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a39c00 x1855007617049088/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074882.241030:0:67486:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67486 initiator: MDT0 00000100:00100000:0.0:1769074882.241054:0:67486:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67486:1855007617049088:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074882.241581:0:67486:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a600 going to sleep for 50 seconds 00000100:00100000:0.0:1769074882.246847:0:67486:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67486:1855007617049088:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074882.246872:0:67486:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67486 initiator: MDT0 00010000:00010000:0.0:1769074882.246885:0:67486:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67486 initiator: MDT0 00010000:00010000:0.0:1769074882.246893:0:67486:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67486 initiator: MDT0 00010000:00010000:0.0:1769074882.246901:0:67486:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 67486 initiator: MDT0 00010000:00010000:0.0:1769074882.246908:0:67486:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 67486 initiator: MDT0 00010000:00010000:0.0:1769074882.246918:0:67486:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035089aa00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074882.246923:0:67486:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67486 initiator: MDT0 00010000:00010000:0.0:1769074882.246929:0:67486:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 67486 initiator: MDT0 00010000:00010000:0.0:1769074882.246936:0:67486:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 67486 initiator: MDT0 00010000:00010000:0.0:1769074882.246943:0:67486:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074882.246949:0:67486:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67486 initiator: MDT0 00010000:00010000:0.0:1769074882.246955:0:67486:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089aa00/0xb7d6bf619879c95f lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 67486 initiator: MDT0 00000002:00100000:0.0:1769074882.246981:0:67486:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0365a39c00 x1855007617049088/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769074937 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074882.246995:0:67486:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1769074882.247152:0:67486:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074882.247175:0:67486:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074882.247188:0:67486:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074882.247199:0:67486:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074882.247207:0:67486:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074882.247213:0:67486:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074882.247217:0:67486:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:0.0:1769074882.247222:0:67486:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641da7bf8 00010000:00010000:0.0:1769074882.247225:0:67486:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074882.247530:0:67486:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074882.247546:0:67486:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074882.247555:0:67486:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=267 00000080:00200000:0.0:1769074882.247562:0:67486:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 44 00000080:00200008:0.0:1769074882.247569:0:67486:0:(file.c:1001:ll_file_open()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00000080:00200080:0.0:1769074882.247749:0:67486:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908) cmd=c0086915 arg=196e3d0 00000100:00100000:0.0:1769074882.247814:0:67486:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67486:1855007617049216:192.168.204.160@tcp:34:lfs.0 00000100:00100000:0.0:1769074882.247951:0:67486:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a600 going to sleep for 50 seconds 00000100:00100000:0.0:1769074882.251922:0:67486:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67486:1855007617049216:192.168.204.160@tcp:34:lfs.0 00000080:00200008:0.0:1769074882.252446:0:67486:0:(file.c:424:ll_file_release()) START file d204c.sanity-flr:[0x200000405:0x209:0x0](ffff9b0355ce0908), flags 304000 00010000:00010000:0.0:1769074882.252480:0:67486:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b800/0xb7d6bf619879c8e1 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x209:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55a75 expref: -99 pid: 67448 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074883.028511:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074884.052405:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1769074884.182097:0:8:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x200:0x0] ignore/verify layout 1/0, layout version 8 restore needed 0 00020000:00200000:0.0:1769074884.182117:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:0.0:1769074884.182129:0:8:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:0.0:1769074884.182162:0:8:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1769074887.125539:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074888.149358:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1769074888.592548:0:67905:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074888.592580:0:67905:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074888.592599:0:67905:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.592617:0:67905:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074888.592625:0:67905:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074888.592636:0:67905:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074888.592640:0:67905:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074888.592646:0:67905:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641affb58 00010000:00010000:2.0:1769074888.592650:0:67905:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.592657:0:67905:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074888.592672:0:67905:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074888.592679:0:67905:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=24 00000080:00200000:2.0:1769074888.592683:0:67905:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:2.0:1769074888.592692:0:67905:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), flags=8 00000080:00200000:2.0:1769074888.592697:0:67905:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908), intent=getattr 00000002:00010000:2.0:1769074888.592711:0:67905:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204c.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0x9:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074888.592763:0:67905:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 67905 initiator: MDT0 00010000:00010000:2.0:1769074888.592772:0:67905:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67905 initiator: MDT0 00010000:00010000:2.0:1769074888.592783:0:67905:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 67905 initiator: MDT0 00000100:00100000:2.0:1769074888.592812:0:67905:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67905:1855007617049984:192.168.204.160@tcp:101:rm.0 00000100:00100000:2.0:1769074888.592926:0:67905:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074888.602683:0:67905:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944137 gen 1 00000100:00100000:2.0:1769074888.602703:0:67905:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67905:1855007617049984:192.168.204.160@tcp:101:rm.0 00010000:00010000:2.0:1769074888.602723:0:67905:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0x9:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.602748:0:67905:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.602756:0:67905:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.602767:0:67905:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.602778:0:67905:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.602789:0:67905:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.602795:0:67905:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.602802:0:67905:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074888.602812:0:67905:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b03482c7800 x1855007617049984/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/1032 e 0 to 0 dl 1769074943 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074888.602824:0:67905:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 448 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1769074888.602838:0:67905:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074888.602852:0:67905:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/522 (0 0) 00000080:00200000:2.0:1769074888.602859:0:67905:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x9:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1769074888.602907:0:67905:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20a:0x0], updating i_size 2097152 00000080:00200000:2.0:1769074888.602909:0:67905:0:(namei.c:151:ll_iget()) got inode: [0x200000405:0x20a:0x0](ffff9b0355ce3248): rc = 0 00000080:00200000:2.0:1769074888.603143:0:67905:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x20a:0x0]: layout version change: 9 -> 9 00000080:00010000:2.0:1769074888.603149:0:67905:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c966 00000080:00010000:2.0:1769074888.603171:0:67905:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) 00000080:00010000:2.0:1769074888.603174:0:67905:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c966 from it ffffb43641affb20 00010000:00010000:2.0:1769074888.603177:0:67905:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074888.603185:0:67905:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 3 00000080:00200008:2.0:1769074888.603225:0:67905:0:(file.c:6252:ll_getattr_dentry()) START file f204c.sanity-flr:[0x200000405:0x20a:0x0](ffff9b0355ce3248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074888.603577:0:67905:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248),name=f204c.sanity-flr 00000002:00010000:2.0:1769074888.603603:0:67905:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20a:0x0]) in obj [0x200000bd0:0x9:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074888.603613:0:67905:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074888.603625:0:67905:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 2/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074888.603632:0:67905:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0x9:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074888.603637:0:67905:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) 00000080:00010000:2.0:1769074888.603640:0:67905:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20a:0x0](ffff9b0355ce3248) for lock 0xb7d6bf619879c966 00000080:00010000:2.0:1769074888.603644:0:67905:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c966 from it ffffb43641affcb8 00010000:00010000:2.0:1769074888.603648:0:67905:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074888.603655:0:67905:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074888.603705:0:67905:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074888.603713:0:67905:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074888.603720:0:67905:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.603728:0:67905:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074888.603734:0:67905:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074888.603738:0:67905:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074888.603741:0:67905:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074888.603744:0:67905:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb43641affc10 00010000:00010000:2.0:1769074888.603747:0:67905:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.603755:0:67905:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074888.603763:0:67905:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074888.603769:0:67905:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204c.sanity-flr, flags=24 00000080:00200000:2.0:1769074888.603774:0:67905:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 1 00000080:00200000:2.0:1769074888.603780:0:67905:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204c.sanity-flr, flags=0 00000080:00200000:2.0:1769074888.603783:0:67905:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x209:0x0](ffff9b0355ce0908), inode mode 41ed mask 3 00000080:00200000:2.0:1769074888.603787:0:67905:0:(namei.c:2354:ll_unlink()) VFS Op:name=f204c.sanity-flr, dir=[0x200000405:0x209:0x0](ffff9b0355ce0908) 00010000:00010000:2.0:1769074888.603811:0:67905:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b600/0xb7d6bf619879c93c lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55ba2 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074888.603825:0:67905:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x20a:0x0]: losing layout lock 00010000:00010000:2.0:1769074888.603836:0:67905:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074888.603844:0:67905:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x20a:0x0]: losing layout lock 00010000:00010000:2.0:1769074888.603850:0:67905:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55a13 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.603891:0:67905:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b600/0xb7d6bf619879c93c lrc: 1/0,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55ba2 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074888.603898:0:67905:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074888.603904:0:67905:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 1/0,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55a13 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074888.603909:0:67905:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:2.0:1769074888.603911:0:67905:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b600/0xb7d6bf619879c93c lrc: 0/0,0 mode: --/CR res: [0x200000405:0x20a:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55ba2 expref: -99 pid: 67463 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074888.604023:0:67905:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c966 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55c90 expref: -99 pid: 67905 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074888.604038:0:67905:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879c8c5 lrc: 0/0,0 mode: --/PR res: [0x200000405:0x20a:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55a13 expref: -99 pid: 67421 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074888.604058:0:67905:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074888.604069:0:67905:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7800 x1855007617050240/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 512/2128 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074888.604096:0:67905:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67905:1855007617050240:192.168.204.160@tcp:36:rm.0 00000100:00100000:2.0:1769074888.604209:0:67905:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074888.613963:0:67905:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:67905:1855007617050240:192.168.204.160@tcp:36:rm.0 00000080:00200000:2.0:1769074888.614023:0:67905:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 8 restore needed 0 00020000:00200000:2.0:1769074888.614038:0:67905:0:(lov_io.c:458:lov_io_mirror_init()) [0x200000405:0x20a:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074888.614041:0:67905:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074888.614048:0:67905:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x10 00020000:00200000:2.0:1769074888.614058:0:67905:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00000080:00200000:2.0:1769074888.614083:0:67905:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074888.614090:0:67905:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x200000405:0x20a:0x0](ffff9b0355ce3248) 00010000:00010000:2.0:1769074889.174608:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074890.195958:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074890.357815:0:2408:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9b0342ad2800: committing for last_committed 17179874640 gen 2 00000100:00100000:2.0:1769074890.357846:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617050880:192.168.204.160@tcp:400:kworker.0 00000001:02000400:0.0:1769074890.655671:0:68051:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204d: FLR write/stale/resync sel test with repeated comp ========================================================== 04:41:28 (1769074888) 00010000:00010000:2.0:1769074891.228283:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074893.341352:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074893.348048:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625260672 00000100:00100000:0.0:1769074893.348070:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625260672 00000100:00100000:0.0:1769074893.348159:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8954 00000100:00100000:0.0:1769074893.348169:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cea300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:36516:x1855007625260672:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074893.348190:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074893.354099:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074893.354131:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cea300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:36516:x1855007625260672:12345-192.168.204.160@tcp:104: Request processed in 5949us (12782us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074893.354147:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8954 00010000:00010000:2.0:1769074893.354275:0:39339:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074893.354287:0:39339:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074893.354300:0:39339:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919ca00 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:2.0:1769074893.354309:0:39339:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x428400030000 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074893.354381:0:39339:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074893.354389:0:39339:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074893.354425:0:39339:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074893.354432:0:39339:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035919ca00/0xb7d6bf619879c935 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55b94 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074893.354461:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074893.354463:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074893.354465:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074893.354475:0:2408:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074893.354494:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617051008:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074893.359157:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074893.359546:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074893.359605:0:39329:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1769074893.359625:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625261056 00000100:00100000:1.0:1769074893.359637:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625261056 00000100:00100000:1.0:1769074893.359829:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625261184 00000100:00100000:1.0:1769074893.359833:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625261184 00000100:00100000:1.0:1769074893.359849:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8955 00000100:00100000:1.0:1769074893.359858:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:8398:x1855007625261056:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074893.359878:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360195:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074893.360203:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:8398:x1855007625261056:12345-192.168.204.160@tcp:104: Request processed in 345us (1063us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074893.360211:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8955 00000100:00100000:1.0:1769074893.360229:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8956 00000100:00100000:1.0:1769074893.360259:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0344e74c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:28955:x1855007625261184:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074893.360267:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360296:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1769074893.360301:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0344e74c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:28955:x1855007625261184:12345-192.168.204.160@tcp:104: Request processed in 43us (756us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074893.360307:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8956 00010000:00010000:1.0:1769074893.360349:0:57730:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1769074893.360357:0:57730:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360366:0:57730:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919c000 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:1.0:1769074893.360372:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x428400010000 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360453:0:57730:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360461:0:57730:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074893.360484:0:57730:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360493:0:57730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c92e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x2cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55b86 expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360515:0:57730:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769074893.360518:0:57730:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360524:0:57730:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919dc00 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:1.0:1769074893.360528:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x428400010000 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360542:0:57730:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360864:0:57730:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074893.360872:0:57730:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.360881:0:57730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035919dc00/0xb7d6bf619879c927 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x2cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55b7f expref: -99 pid: 67463 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074893.361029:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074893.361031:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074893.361033:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074893.361054:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617051136:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074893.361145:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1769074893.361152:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617051264:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074893.383441:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617051264:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074893.383458:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617051136:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074893.390119:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617051008:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074894.292163:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074895.315626:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074895.444969:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617051520:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074895.445129:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617051776:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074895.445172:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1769074895.445179:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617051392:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074895.445207:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617051648:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074895.445580:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617051904:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074895.462789:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617051392:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074895.462821:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617051648:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074895.462833:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617051904:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074895.462844:0:2407:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944138 gen 1 00000100:00100000:1.0:1769074895.462867:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617051520:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074895.462879:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617051776:192.168.204.160@tcp:400:kworker.0 00010000:00010000:1.0:1769074899.411684:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074900.563943:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617052032:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074900.564078:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617052288:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074900.564107:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617052544:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074900.564135:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074900.564140:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617052160:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074900.571565:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617052416:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074900.578302:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617052160:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074900.578322:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617052416:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074900.578332:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617052032:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074900.578343:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617052288:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074900.578353:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617052544:192.168.204.160@tcp:400:kworker.0 00010000:00010000:2.0:1769074901.460131:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074902.483506:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074906.579516:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1769074907.131651:0:68433:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.131684:0:68433:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.131703:0:68433:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.131720:0:68433:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.131728:0:68433:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.131736:0:68433:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.131740:0:68433:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074907.131746:0:68433:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb4364235fb58 00010000:00010000:2.0:1769074907.131751:0:68433:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.131759:0:68433:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.131775:0:68433:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074907.131791:0:68433:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d204d.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), flags=13 00000080:00200000:2.0:1769074907.131795:0:68433:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d204d.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=getattr 00000002:00010000:2.0:1769074907.131877:0:68433:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.131926:0:68433:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.131933:0:68433:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.131941:0:68433:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68433 initiator: MDT0 00000100:00100000:2.0:1769074907.131967:0:68433:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68433:1855007617053440:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074907.132219:0:68433:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 50 seconds 00000100:00100000:2.0:1769074907.135335:0:68433:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68433:1855007617053440:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074907.135359:0:68433:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.135371:0:68433:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.135382:0:68433:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.135389:0:68433:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.135397:0:68433:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.135402:0:68433:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860400 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074907.135407:0:68433:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.135413:0:68433:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.135420:0:68433:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.135427:0:68433:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074907.135432:0:68433:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68433 initiator: MDT0 00010000:00010000:2.0:1769074907.135437:0:68433:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c96d lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68433 initiator: MDT0 00000002:00100000:2.0:1769074907.135454:0:68433:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9b03482c6d80 x1855007617053440/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/584 e 0 to 0 dl 1769074962 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074907.135467:0:68433:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1769074907.135493:0:68433:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.135509:0:68433:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00010000:2.0:1769074907.135518:0:68433:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb4364235f940 00010000:00010000:2.0:1769074907.135521:0:68433:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.135528:0:68433:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.135615:0:68433:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.135630:0:68433:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.135638:0:68433:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.135645:0:68433:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.135651:0:68433:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.135655:0:68433:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.135658:0:68433:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074907.135663:0:68433:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb4364235fbe8 00010000:00010000:2.0:1769074907.135666:0:68433:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.135673:0:68433:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.135685:0:68433:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074907.135692:0:68433:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=16392 00000080:00200008:2.0:1769074907.135717:0:68433:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074907.135723:0:68433:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.135729:0:68433:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.135735:0:68433:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.135742:0:68433:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.135748:0:68433:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.135752:0:68433:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.135754:0:68433:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074907.135758:0:68433:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb4364235fcb8 00010000:00010000:2.0:1769074907.135761:0:68433:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.135767:0:68433:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200008:2.0:1769074907.137809:0:68433:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074907.137819:0:68433:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.137839:0:68433:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.137861:0:68433:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.137873:0:68433:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.137880:0:68433:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.137886:0:68433:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.137889:0:68433:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074907.137899:0:68433:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb4364235fcb8 00010000:00010000:2.0:1769074907.137904:0:68433:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.137912:0:68433:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.138077:0:68433:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.138086:0:68433:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.138092:0:68433:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.138100:0:68433:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.138106:0:68433:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.138110:0:68433:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.138112:0:68433:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c87f 00000080:00010000:2.0:1769074907.138115:0:68433:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c87f from it ffffb4364235fbb0 00010000:00010000:2.0:1769074907.138118:0:68433:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.138126:0:68433:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.138138:0:68433:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:2.0:1769074907.138146:0:68433:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:2.0:1769074907.138162:0:68433:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=c03066f0 arg=7ffe995b2870 00000080:00200000:2.0:1769074907.138175:0:68433:0:(dir.c:483:ll_dir_setdirstripe()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) name=d204d.sanity-flr stripe_offset=0 stripe_count=2, hash_type=3 00010000:00010000:2.0:1769074907.138201:0:68433:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.138281:0:68433:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.138288:0:68433:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074907.138291:0:68433:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348215c00/0xb7d6bf619879c87f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55933 expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00000020:00100000:2.0:1769074907.138327:0:68433:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074907.138341:0:68433:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c6d80 x1855007617053696/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 568/2080 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074907.138365:0:68433:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68433:1855007617053696:192.168.204.160@tcp:36:lfs.0 00000100:00100000:2.0:1769074907.138461:0:68433:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 50 seconds 00000100:00100000:2.0:1769074907.164538:0:68433:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68433:1855007617053696:192.168.204.160@tcp:36:lfs.0 00000080:00200000:2.0:1769074907.164615:0:68433:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988) 00000080:00200000:2.0:1769074907.164623:0:68433:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20c:0x0], updating i_size 4096 00000080:00200008:2.0:1769074907.164657:0:68433:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000100:00100000:0.0:1769074907.306009:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074907.310779:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625269120 00000100:00100000:0.0:1769074907.310796:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625269120 00000100:00100000:0.0:1769074907.310874:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8957 00000100:00100000:0.0:1769074907.310885:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a39c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6514:x1855007625269120:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074907.310900:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c99e lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd55d85 expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074907.311696:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074907.311708:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a39c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6514:x1855007625269120:12345-192.168.204.160@tcp:104: Request processed in 823us (5708us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074907.311718:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8957 00010000:00010000:2.0:1769074907.313064:0:60112:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c99e lrc: 2/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd55d85 expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.313076:0:60112:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074907.313113:0:60112:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c99e lrc: 1/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd55d85 expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.313121:0:60112:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879c99e lrc: 0/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd55d85 expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.313137:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074907.313139:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074907.313140:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074907.313149:0:2408:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074907.313166:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617055104:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074907.320438:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617055104:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074907.344025:0:68436:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a600 going to sleep for 11 seconds 00000100:00100000:2.0:1769074907.350833:0:68436:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68436:1855007617055616:192.168.204.160@tcp:35:lfs.0 00000002:00100000:2.0:1769074907.350883:0:68436:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0352a18a80 x1855007617054720/t4294968373(4294968373) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 536/608 e 0 to 0 dl 1769074962 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074907.363160:0:68437:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.363200:0:68437:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.363219:0:68437:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.363618:0:68437:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.363629:0:68437:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.363638:0:68437:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.363642:0:68437:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074907.363649:0:68437:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642387bf8 00010000:00010000:2.0:1769074907.363653:0:68437:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.363749:0:68437:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.363765:0:68437:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074907.363773:0:68437:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:2.0:1769074907.363993:0:68437:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:2.0:1769074907.364007:0:68437:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b03586efc80, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074907.364015:0:68437:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open 00000002:00010000:2.0:1769074907.364035:0:68437:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074907.364094:0:68437:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.364102:0:68437:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68437 initiator: MDT0 00000020:00100000:2.0:1769074907.364113:0:68437:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074907.364126:0:68437:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c5880 x1855007617055872/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074907.364136:0:68437:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68437 initiator: MDT0 00000100:00100000:2.0:1769074907.364156:0:68437:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68437:1855007617055872:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074907.364603:0:68437:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 50 seconds 00000100:00100000:2.0:1769074907.368917:0:68437:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68437:1855007617055872:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074907.368940:0:68437:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.368953:0:68437:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.368960:0:68437:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.368967:0:68437:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.368974:0:68437:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.368980:0:68437:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860400 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074907.368985:0:68437:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.368991:0:68437:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.368997:0:68437:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 5/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.369004:0:68437:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074907.369010:0:68437:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.369015:0:68437:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9ac lrc: 0/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68437 initiator: MDT0 00000002:00100000:2.0:1769074907.369043:0:68437:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b03482c5880 x1855007617055872/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/608 e 0 to 0 dl 1769074962 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074907.369057:0:68437:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x240000405:0xa:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074907.369327:0:68437:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.369350:0:68437:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.369363:0:68437:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.369374:0:68437:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.369381:0:68437:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.369386:0:68437:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.369390:0:68437:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074907.369394:0:68437:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642387bf8 00010000:00010000:2.0:1769074907.369397:0:68437:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.369405:0:68437:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.369419:0:68437:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074907.369426:0:68437:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:2.0:1769074907.369433:0:68437:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 44 00000080:00200008:2.0:1769074907.369440:0:68437:0:(file.c:1001:ll_file_open()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000002:00010000:2.0:1769074907.369454:0:68437:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20c:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:2.0:1769074907.369502:0:68437:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.369509:0:68437:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68437 initiator: MDT0 00000020:00100000:2.0:1769074907.369520:0:68437:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074907.369526:0:68437:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c5880 x1855007617056128/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074907.369535:0:68437:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68437 initiator: MDT0 00000100:00100000:2.0:1769074907.369554:0:68437:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68437:1855007617056128:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074907.370069:0:68437:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 50 seconds 00000100:00100000:2.0:1769074907.374921:0:68437:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68437:1855007617056128:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074907.374944:0:68437:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.374957:0:68437:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.374964:0:68437:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.374970:0:68437:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.374978:0:68437:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.374984:0:68437:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860200 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074907.374988:0:68437:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.374994:0:68437:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.375004:0:68437:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.375011:0:68437:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074907.375018:0:68437:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68437 initiator: MDT0 00010000:00010000:2.0:1769074907.375024:0:68437:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9b3 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68437 initiator: MDT0 00000002:00100000:2.0:1769074907.375043:0:68437:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9b03482c5880 x1855007617056128/t8589944140(8589944140) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074962 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074907.375057:0:68437:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b03482c5880 x1855007617056128/t8589944140(8589944140) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074962 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074907.375073:0:68437:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074907.375107:0:68437:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20c:0x0], updating i_size 4096 00000002:00100000:2.0:1769074907.375126:0:68437:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b03482c5880 x1855007617056128/t8589944140(8589944140) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074962 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1769074907.375429:0:68437:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988) cmd=c0086915 arg=24993d0 00000100:00100000:2.0:1769074907.375493:0:68437:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68437:1855007617056256:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074907.375761:0:68437:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 50 seconds 00000100:00100000:2.0:1769074907.380077:0:68437:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68437:1855007617056256:192.168.204.160@tcp:34:lfs.0 00000080:00200008:2.0:1769074907.380535:0:68437:0:(file.c:424:ll_file_release()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000002:00100000:2.0:1769074907.380610:0:68437:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b03482c5880 x1855007617056128/t8589944140(8589944140) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074962 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074907.380623:0:68437:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074907.380629:0:68437:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c5500 x1855007617056512/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074907.380652:0:68437:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68437:1855007617056512:192.168.204.160@tcp:35:lfs.0 00000100:00100000:2.0:1769074907.380879:0:68437:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 11 seconds 00000100:00100000:2.0:1769074907.383524:0:68437:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68437:1855007617056512:192.168.204.160@tcp:35:lfs.0 00000002:00100000:2.0:1769074907.383558:0:68437:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b03482c5880 x1855007617056128/t8589944140(8589944140) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074962 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1769074907.415578:0:68441:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074907.415588:0:68441:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.415613:0:68441:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.415632:0:68441:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.415648:0:68441:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.415742:0:68441:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.415750:0:68441:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.415753:0:68441:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074907.415760:0:68441:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642397cb8 00010000:00010000:2.0:1769074907.415763:0:68441:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.415772:0:68441:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.415865:0:68441:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.415873:0:68441:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.415880:0:68441:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.415888:0:68441:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.415894:0:68441:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.415898:0:68441:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.415900:0:68441:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074907.415903:0:68441:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642397bb0 00010000:00010000:2.0:1769074907.415905:0:68441:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.415912:0:68441:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.415919:0:68441:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:2.0:1769074907.415926:0:68441:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:2.0:1769074907.415943:0:68441:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffedda09b4c 00000080:00200008:2.0:1769074907.415958:0:68441:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:2.0:1769074907.415978:0:68441:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.415985:0:68441:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.415991:0:68441:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.415998:0:68441:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.416004:0:68441:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.416008:0:68441:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.416010:0:68441:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074907.416013:0:68441:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642397bb0 00010000:00010000:2.0:1769074907.416016:0:68441:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.416023:0:68441:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.416031:0:68441:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:2.0:1769074907.416035:0:68441:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 100000 00000080:00200080:2.0:1769074907.416043:0:68441:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=8080667f arg=7ffedda08820 00000080:00200008:2.0:1769074907.417170:0:68441:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 100000 00000080:00200008:2.0:1769074907.417644:0:68441:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1769074907.417649:0:68441:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074907.417750:0:68441:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074907.417763:0:68441:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.417777:0:68441:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074907.417784:0:68441:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074907.417790:0:68441:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074907.417794:0:68441:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074907.417800:0:68441:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642397cb8 00010000:00010000:2.0:1769074907.417803:0:68441:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.417816:0:68441:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074907.603473:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074908.627639:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074910.803681:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617056640:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074910.803797:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617056896:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074910.803826:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617057152:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074910.803853:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1769074910.803858:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617056768:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074910.803877:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617057024:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074910.808548:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617056768:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074910.808589:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617057024:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074910.808611:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617056640:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074910.808624:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617056896:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074910.808636:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617057152:192.168.204.160@tcp:400:kworker.0 00010000:00010000:1.0:1769074912.724600:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074913.751848:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074914.777668:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074915.796193:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074915.927721:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617057408:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074915.933044:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617057664:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074915.942032:0:2409:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944141 gen 1 00000100:00100000:2.0:1769074915.942065:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617057408:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074915.942084:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617057664:192.168.204.160@tcp:400:kworker.0 00000080:00200000:1.0:1769074919.252069:0:68465:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074919.252112:0:68465:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074919.252135:0:68465:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074919.252155:0:68465:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074919.252164:0:68465:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074919.252174:0:68465:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074919.252178:0:68465:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074919.252186:0:68465:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb436423d7bf8 00010000:00010000:1.0:1769074919.252190:0:68465:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074919.252198:0:68465:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074919.252219:0:68465:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074919.252231:0:68465:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=25 00000080:00200000:1.0:1769074919.252273:0:68465:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:1.0:1769074919.252289:0:68465:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 3 00000080:00200000:1.0:1769074919.252295:0:68465:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b03592e7980, open_flags 8042, mode 81b6 opened 0 00000080:00200000:1.0:1769074919.252306:0:68465:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open|creat 00000002:00010000:1.0:1769074919.252338:0:68465:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x240000404:0x4e:0x0]) in obj [0x240000405:0xa:0x0], intent: open|creat flags 0100103 00010000:00010000:1.0:1769074919.252915:0:68465:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919d000/0xb7d6bf619879c9ba lrc: 3/0,1 mode: --/CW res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68465 initiator: MDT0 00010000:00010000:1.0:1769074919.252925:0:68465:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919d000/0xb7d6bf619879c9ba lrc: 3/0,1 mode: --/CW res: [0x240000405:0xa:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68465 initiator: MDT0 00000020:00100000:1.0:1769074919.252941:0:68465:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074919.252956:0:68465:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ceb480 x1855007617058048/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/67888 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074919.252965:0:68465:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919d000/0xb7d6bf619879c9ba lrc: 3/0,1 mode: --/CW res: [0x240000405:0xa:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68465 initiator: MDT0 00000100:00100000:1.0:1769074919.252991:0:68465:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68465:1855007617058048:192.168.204.160@tcp:101:dd.0 00000100:00100000:1.0:1769074919.253295:0:68465:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a68240 going to sleep for 50 seconds 00000080:00010000:2.0:1769074919.335919:0:68465:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879c9cf 00000080:00010000:2.0:1769074919.335931:0:68465:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c9cf from it ffffb436423d78f8 00010000:00010000:2.0:1769074919.335937:0:68465:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005ac00/0xb7d6bf619879c9cf lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55e1f expref: -99 pid: 68465 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074919.335947:0:68465:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005ac00/0xb7d6bf619879c9cf lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e1f expref: -99 pid: 68465 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074919.336005:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16896 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336108:0:68465:0:(rw26.c:912:ll_write_end()) pos 69206016, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336120:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a66ad0a1[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336123:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000000f0a6aec 17ffffc0001001 3:0 ffff9b0353099510 16896 lru 00000080:00200000:2.0:1769074919.336131:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000077ada5c8 16896: 1< 0 - - > 2< 69206016 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336133:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a66ad0a1 00000080:00200000:2.0:1769074919.336134:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 1. 00000080:00200000:2.0:1769074919.336138:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16897 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336157:0:68465:0:(rw26.c:912:ll_write_end()) pos 69210112, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336161:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000fe465556[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336165:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000fcf8c9ca 17ffffc0001001 3:0 ffff9b0353099948 16897 lru 00000080:00200000:2.0:1769074919.336170:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008db7963d 16897: 1< 0 - - > 2< 69210112 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336172:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000fe465556 00000080:00200000:2.0:1769074919.336173:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 2. 00000080:00200000:2.0:1769074919.336175:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16898 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336193:0:68465:0:(rw26.c:912:ll_write_end()) pos 69214208, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336198:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004a78f575[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336201:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000009ca02ceb 17ffffc0001001 3:0 ffff9b0353099000 16898 lru 00000080:00200000:2.0:1769074919.336206:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000b94909c8 16898: 1< 0 - - > 2< 69214208 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336208:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004a78f575 00000080:00200000:2.0:1769074919.336209:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 3. 00000080:00200000:2.0:1769074919.336211:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16899 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336229:0:68465:0:(rw26.c:912:ll_write_end()) pos 69218304, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336255:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000007227ec2[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336258:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000005c7f37f 17ffffc0001001 3:0 ffff9b0353099ca8 16899 lru 00000080:00200000:2.0:1769074919.336263:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000a0cead7c 16899: 1< 0 - - > 2< 69218304 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336265:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000007227ec2 00000080:00200000:2.0:1769074919.336266:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 4. 00000080:00200000:2.0:1769074919.336269:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16900 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336289:0:68465:0:(rw26.c:912:ll_write_end()) pos 69222400, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336293:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000098f33ff[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336296:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000dd0adada 17ffffc0001001 3:0 ffff9b03530995e8 16900 lru 00000080:00200000:2.0:1769074919.336301:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000005ffb3c57 16900: 1< 0 - - > 2< 69222400 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336303:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000098f33ff 00000080:00200000:2.0:1769074919.336304:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 5. 00000080:00200000:2.0:1769074919.336306:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16901 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336326:0:68465:0:(rw26.c:912:ll_write_end()) pos 69226496, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336330:0:68465:0:(rw26.c:969:ll_write_end()) page@000000003dac592f[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336332:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000e1fa3a35 17ffffc0001001 3:0 ffff9b03530990d8 16901 lru 00000080:00200000:2.0:1769074919.336337:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000059e41e5b 16901: 1< 0 - - > 2< 69226496 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336339:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000003dac592f 00000080:00200000:2.0:1769074919.336340:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 6. 00000080:00200000:2.0:1769074919.336342:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16902 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336356:0:68465:0:(rw26.c:912:ll_write_end()) pos 69230592, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336360:0:68465:0:(rw26.c:969:ll_write_end()) page@000000009e2235ac[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336363:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000640b9a84 17ffffc0001001 3:0 ffff9b0353099288 16902 lru 00000080:00200000:2.0:1769074919.336367:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000fc42f0d4 16902: 1< 0 - - > 2< 69230592 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336369:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000009e2235ac 00000080:00200000:2.0:1769074919.336370:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 7. 00000080:00200000:2.0:1769074919.336372:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16903 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336388:0:68465:0:(rw26.c:912:ll_write_end()) pos 69234688, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336392:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006bf06eaa[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336394:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b6bba6bf 17ffffc0001001 3:0 ffff9b0353099870 16903 lru 00000080:00200000:2.0:1769074919.336398:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f3e9d88c 16903: 1< 0 - - > 2< 69234688 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336401:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006bf06eaa 00000080:00200000:2.0:1769074919.336402:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 8. 00000080:00200000:2.0:1769074919.336403:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16904 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336416:0:68465:0:(rw26.c:912:ll_write_end()) pos 69238784, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336419:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000096931e00[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336421:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000073381bc3 17ffffc0001001 3:0 ffff9b0353099bd0 16904 lru 00000080:00200000:2.0:1769074919.336425:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000004638403d 16904: 1< 0 - - > 2< 69238784 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336427:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000096931e00 00000080:00200000:2.0:1769074919.336427:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 9. 00000080:00200000:2.0:1769074919.336429:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16905 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336442:0:68465:0:(rw26.c:912:ll_write_end()) pos 69242880, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336445:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a8a66e7d[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336448:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000a78c7b06 17ffffc0001001 3:0 ffff9b03530996c0 16905 lru 00000080:00200000:2.0:1769074919.336453:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000004347737 16905: 1< 0 - - > 2< 69242880 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336455:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a8a66e7d 00000080:00200000:2.0:1769074919.336457:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 10. 00000080:00200000:2.0:1769074919.336458:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16906 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336473:0:68465:0:(rw26.c:912:ll_write_end()) pos 69246976, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336477:0:68465:0:(rw26.c:969:ll_write_end()) page@000000009cc8368b[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336479:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000016c0d689 17ffffc0001001 3:0 ffff9b0353099438 16906 lru 00000080:00200000:2.0:1769074919.336483:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000009fbdca2a 16906: 1< 0 - - > 2< 69246976 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336485:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000009cc8368b 00000080:00200000:2.0:1769074919.336486:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 11. 00000080:00200000:2.0:1769074919.336488:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16907 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336503:0:68465:0:(rw26.c:912:ll_write_end()) pos 69251072, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336507:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a7539ecc[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336509:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000656fcbf7 17ffffc0001001 3:0 ffff9b0353099e58 16907 lru 00000080:00200000:2.0:1769074919.336599:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000082bd0760 16907: 1< 0 - - > 2< 69251072 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336604:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a7539ecc 00000080:00200000:2.0:1769074919.336606:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 12. 00000080:00200000:2.0:1769074919.336609:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16908 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336627:0:68465:0:(rw26.c:912:ll_write_end()) pos 69255168, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336631:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004496ba81[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336634:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d292a9c9 17ffffc0001001 3:0 ffff9b0353099d80 16908 lru 00000080:00200000:2.0:1769074919.336649:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000bcbc9ad0 16908: 1< 0 - - > 2< 69255168 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336652:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004496ba81 00000080:00200000:2.0:1769074919.336653:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 13. 00000080:00200000:2.0:1769074919.336655:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16909 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336671:0:68465:0:(rw26.c:912:ll_write_end()) pos 69259264, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336675:0:68465:0:(rw26.c:969:ll_write_end()) page@000000008edf76d7[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336678:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000008c4909af 17ffffc0001001 3:0 ffff9b0353099af8 16909 lru 00000080:00200000:2.0:1769074919.336683:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000038ed07d4 16909: 1< 0 - - > 2< 69259264 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336687:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000008edf76d7 00000080:00200000:2.0:1769074919.336688:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 14. 00000080:00200000:2.0:1769074919.336690:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16910 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336717:0:68465:0:(rw26.c:912:ll_write_end()) pos 69263360, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336722:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000d6e23ac6[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336724:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000856951c0 17ffffc0001021 2:0 ffff9b0353099360 16910 lru 00000080:00200000:2.0:1769074919.336729:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000b45927ae 16910: 1< 0 - - > 2< 69263360 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336731:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000d6e23ac6 00000080:00200000:2.0:1769074919.336732:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 15. 00000080:00200000:2.0:1769074919.336734:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16911 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336750:0:68465:0:(rw26.c:912:ll_write_end()) pos 69267456, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336754:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000991366ed[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336756:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000aafa8c51 17ffffc0001001 3:0 ffff9b0353099a20 16911 lru 00000080:00200000:2.0:1769074919.336761:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d7bd1412 16911: 1< 0 - - > 2< 69267456 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336764:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000991366ed 00000080:00200000:2.0:1769074919.336765:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 16. 00000080:00200000:2.0:1769074919.336767:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16912 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336781:0:68465:0:(rw26.c:912:ll_write_end()) pos 69271552, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336785:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a7c2d2a2[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336787:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000003489e3bd 17ffffc0001001 3:0 ffff9b03530991b0 16912 lru 00000080:00200000:2.0:1769074919.336792:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000567fe901 16912: 1< 0 - - > 2< 69271552 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336794:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a7c2d2a2 00000080:00200000:2.0:1769074919.336795:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 17. 00000080:00200000:2.0:1769074919.336796:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16913 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336812:0:68465:0:(rw26.c:912:ll_write_end()) pos 69275648, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336816:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006928dd56[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336818:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000057f6fd67 17ffffc0001001 3:0 ffff9b0353099798 16913 lru 00000080:00200000:2.0:1769074919.336824:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000423444c0 16913: 1< 0 - - > 2< 69275648 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336826:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006928dd56 00000080:00200000:2.0:1769074919.336827:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 18. 00000080:00200000:2.0:1769074919.336829:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16914 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336849:0:68465:0:(rw26.c:912:ll_write_end()) pos 69279744, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336853:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000022d59f86[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336856:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000f0e88154 17ffffc0001001 3:0 ffff9b03519905e8 16914 lru 00000080:00200000:2.0:1769074919.336861:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000002b33520d 16914: 1< 0 - - > 2< 69279744 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336863:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000022d59f86 00000080:00200000:2.0:1769074919.336864:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 19. 00000080:00200000:2.0:1769074919.336866:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16915 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336882:0:68465:0:(rw26.c:912:ll_write_end()) pos 69283840, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336885:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000091697a0b[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336888:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000c7d54bdd 17ffffc0001001 3:0 ffff9b03519900d8 16915 lru 00000080:00200000:2.0:1769074919.336892:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000001f7a2e6b 16915: 1< 0 - - > 2< 69283840 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336894:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000091697a0b 00000080:00200000:2.0:1769074919.336895:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 20. 00000080:00200000:2.0:1769074919.336897:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16916 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336911:0:68465:0:(rw26.c:912:ll_write_end()) pos 69287936, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336915:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000ee1dea40[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336917:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b8127f4e 17ffffc0001001 3:0 ffff9b0351990288 16916 lru 00000080:00200000:2.0:1769074919.336923:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000439dc334 16916: 1< 0 - - > 2< 69287936 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336925:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000ee1dea40 00000080:00200000:2.0:1769074919.336926:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 21. 00000080:00200000:2.0:1769074919.336928:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16917 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336943:0:68465:0:(rw26.c:912:ll_write_end()) pos 69292032, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336947:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006af44c26[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336949:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000523fbb22 17ffffc0001001 3:0 ffff9b0351990870 16917 lru 00000080:00200000:2.0:1769074919.336954:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000019329ec4 16917: 1< 0 - - > 2< 69292032 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336956:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006af44c26 00000080:00200000:2.0:1769074919.336957:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 22. 00000080:00200000:2.0:1769074919.336959:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16918 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.336973:0:68465:0:(rw26.c:912:ll_write_end()) pos 69296128, len 4096, copied 4096 00000080:00200000:2.0:1769074919.336977:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000003707038[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.336979:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000f6e53161 17ffffc0001001 3:0 ffff9b0351990bd0 16918 lru 00000080:00200000:2.0:1769074919.336984:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000018db6095 16918: 1< 0 - - > 2< 69296128 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.336986:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000003707038 00000080:00200000:2.0:1769074919.336988:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 23. 00000080:00200000:2.0:1769074919.336989:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16919 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.337003:0:68465:0:(rw26.c:912:ll_write_end()) pos 69300224, len 4096, copied 4096 00000080:00200000:2.0:1769074919.337007:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b3328b77[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.337009:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000008426a173 17ffffc0001001 3:0 ffff9b03519906c0 16919 lru 00000080:00200000:2.0:1769074919.337014:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000986700d7 16919: 1< 0 - - > 2< 69300224 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.337016:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b3328b77 00000080:00200000:2.0:1769074919.337017:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 24. 00000080:00200000:2.0:1769074919.337019:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16920 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.337033:0:68465:0:(rw26.c:912:ll_write_end()) pos 69304320, len 4096, copied 4096 00000080:00200000:2.0:1769074919.337037:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000047a05f92[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.337040:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000067eb3a00 17ffffc0001001 3:0 ffff9b0351990360 16920 lru 00000080:00200000:2.0:1769074919.337044:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000061012214 16920: 1< 0 - - > 2< 69304320 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.337046:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000047a05f92 00000080:00200000:2.0:1769074919.337047:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 25. 00000080:00200000:2.0:1769074919.337049:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16921 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.337064:0:68465:0:(rw26.c:912:ll_write_end()) pos 69308416, len 4096, copied 4096 00000080:00200000:2.0:1769074919.337068:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000cd9179ea[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.337071:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000134ae961 17ffffc0001001 3:0 ffff9b0351990a20 16921 lru 00000080:00200000:2.0:1769074919.337076:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d5ef11bd 16921: 1< 0 - - > 2< 69308416 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.337078:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000cd9179ea 00000080:00200000:2.0:1769074919.337079:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 26. 00000080:00200000:2.0:1769074919.337081:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16922 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.337096:0:68465:0:(rw26.c:912:ll_write_end()) pos 69312512, len 4096, copied 4096 00000080:00200000:2.0:1769074919.337099:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000e43c7422[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.337102:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000f6bd437d 17ffffc0001001 3:0 ffff9b03519901b0 16922 lru 00000080:00200000:2.0:1769074919.337107:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000006ef700c8 16922: 1< 0 - - > 2< 69312512 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.337109:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000e43c7422 00000080:00200000:2.0:1769074919.337110:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 27. 00000080:00200000:2.0:1769074919.337111:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16923 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.337127:0:68465:0:(rw26.c:912:ll_write_end()) pos 69316608, len 4096, copied 4096 00000080:00200000:2.0:1769074919.337131:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000c601e707[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.337133:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000bffc22f2 17ffffc0001001 3:0 ffff9b0351990798 16923 lru 00000080:00200000:2.0:1769074919.337139:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000020614bce 16923: 1< 0 - - > 2< 69316608 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.337141:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000c601e707 00000080:00200000:2.0:1769074919.337142:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 28. 00000080:00200000:2.0:1769074919.337144:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16924 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.337156:0:68465:0:(rw26.c:912:ll_write_end()) pos 69320704, len 4096, copied 4096 00000080:00200000:2.0:1769074919.337160:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000090855b1e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.337162:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d0a8d5de 17ffffc0001001 3:0 ffff9b0351990af8 16924 lru 00000080:00200000:2.0:1769074919.337167:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000004cd1c0c3 16924: 1< 0 - - > 2< 69320704 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.337170:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000090855b1e 00000080:00200000:2.0:1769074919.337170:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 29. 00000080:00200000:2.0:1769074919.337173:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16925 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.337190:0:68465:0:(rw26.c:912:ll_write_end()) pos 69324800, len 4096, copied 4096 00000080:00200000:2.0:1769074919.337195:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000390cd6db[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.337198:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000003cf2acda 17ffffc0001021 2:0 ffff9b0351990d80 16925 lru 00000080:00200000:2.0:1769074919.337203:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000057815f53 16925: 1< 0 - - > 2< 69324800 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.337206:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000390cd6db 00000080:00200000:2.0:1769074919.337207:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 30. 00000080:00200000:2.0:1769074919.337209:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16926 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.337227:0:68465:0:(rw26.c:912:ll_write_end()) pos 69328896, len 4096, copied 4096 00000080:00200000:2.0:1769074919.337231:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b63ee691[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.338891:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000040642d5b 17ffffc0001001 3:0 ffff9b0351990e58 16926 lru 00000080:00200000:2.0:1769074919.338902:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000fa4f7c43 16926: 1< 0 - - > 2< 69328896 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.338905:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b63ee691 00000080:00200000:2.0:1769074919.338907:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 31. 00000080:00200000:2.0:1769074919.338916:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16927 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.338976:0:68465:0:(rw26.c:912:ll_write_end()) pos 69332992, len 4096, copied 4096 00000080:00200000:2.0:1769074919.338984:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004a64ac84[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.338988:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000a55abc1c 17ffffc0001001 3:0 ffff9b0351990438 16927 lru 00000080:00200000:2.0:1769074919.338994:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000bfd90450 16927: 1< 0 - - > 2< 69332992 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.338997:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004a64ac84 00000080:00200000:2.0:1769074919.338998:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 32. 00000080:00200000:2.0:1769074919.339000:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16928 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339019:0:68465:0:(rw26.c:912:ll_write_end()) pos 69337088, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339024:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000c10514c6[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339027:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000014233478 17ffffc0001001 3:0 ffff9b0351990510 16928 lru 00000080:00200000:2.0:1769074919.339033:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000004b1f025a 16928: 1< 0 - - > 2< 69337088 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339036:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000c10514c6 00000080:00200000:2.0:1769074919.339037:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 33. 00000080:00200000:2.0:1769074919.339039:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16929 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339056:0:68465:0:(rw26.c:912:ll_write_end()) pos 69341184, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339060:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000144b74c4[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339063:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000a0e25d42 17ffffc0001001 3:0 ffff9b0351990948 16929 lru 00000080:00200000:2.0:1769074919.339068:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000c62d5990 16929: 1< 0 - - > 2< 69341184 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339071:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000144b74c4 00000080:00200000:2.0:1769074919.339072:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 34. 00000080:00200000:2.0:1769074919.339074:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16930 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339091:0:68465:0:(rw26.c:912:ll_write_end()) pos 69345280, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339095:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000c7e20b54[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339098:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000a621e970 17ffffc0001001 3:0 ffff9b0351990000 16930 lru 00000080:00200000:2.0:1769074919.339103:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000007d09da6c 16930: 1< 0 - - > 2< 69345280 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339105:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000c7e20b54 00000080:00200000:2.0:1769074919.339107:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 35. 00000080:00200000:2.0:1769074919.339108:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16931 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339124:0:68465:0:(rw26.c:912:ll_write_end()) pos 69349376, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339127:0:68465:0:(rw26.c:969:ll_write_end()) page@000000008e4ab0a2[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339130:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000018ec07f3 17ffffc0001001 3:0 ffff9b0351990ca8 16931 lru 00000080:00200000:2.0:1769074919.339135:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000608fe81d 16931: 1< 0 - - > 2< 69349376 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339138:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000008e4ab0a2 00000080:00200000:2.0:1769074919.339139:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 36. 00000080:00200000:2.0:1769074919.339141:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16932 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339160:0:68465:0:(rw26.c:912:ll_write_end()) pos 69353472, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339164:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000eeb9cbe8[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339166:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000581f3397 17ffffc0001001 3:0 ffff9b0360b1cca8 16932 lru 00000080:00200000:2.0:1769074919.339171:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000520a8d3b 16932: 1< 0 - - > 2< 69353472 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339173:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000eeb9cbe8 00000080:00200000:2.0:1769074919.339174:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 37. 00000080:00200000:2.0:1769074919.339176:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16933 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339192:0:68465:0:(rw26.c:912:ll_write_end()) pos 69357568, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339196:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000068d4fe68[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339199:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000042401ae4 17ffffc0001001 3:0 ffff9b0360b1c5e8 16933 lru 00000080:00200000:2.0:1769074919.339204:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f1dc545d 16933: 1< 0 - - > 2< 69357568 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339206:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000068d4fe68 00000080:00200000:2.0:1769074919.339208:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 38. 00000080:00200000:2.0:1769074919.339210:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16934 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339227:0:68465:0:(rw26.c:912:ll_write_end()) pos 69361664, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339252:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006b4b68fe[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339256:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000db477fb4 17ffffc0001001 3:0 ffff9b0360b1c0d8 16934 lru 00000080:00200000:2.0:1769074919.339261:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000001f1c6700 16934: 1< 0 - - > 2< 69361664 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339264:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006b4b68fe 00000080:00200000:2.0:1769074919.339265:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 39. 00000080:00200000:2.0:1769074919.339267:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16935 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339287:0:68465:0:(rw26.c:912:ll_write_end()) pos 69365760, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339291:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b826a5b8[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339294:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000c6602d26 17ffffc0001001 3:0 ffff9b0360b1c288 16935 lru 00000080:00200000:2.0:1769074919.339300:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000eadf947f 16935: 1< 0 - - > 2< 69365760 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339303:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b826a5b8 00000080:00200000:2.0:1769074919.339304:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 40. 00000080:00200000:2.0:1769074919.339306:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16936 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339322:0:68465:0:(rw26.c:912:ll_write_end()) pos 69369856, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339327:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000d7a3fb98[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339330:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000eaaec211 17ffffc0001001 3:0 ffff9b0360b1c870 16936 lru 00000080:00200000:2.0:1769074919.339336:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000001403bcdd 16936: 1< 0 - - > 2< 69369856 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339339:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000d7a3fb98 00000080:00200000:2.0:1769074919.339340:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 41. 00000080:00200000:2.0:1769074919.339342:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16937 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339359:0:68465:0:(rw26.c:912:ll_write_end()) pos 69373952, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339363:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b1e58bfd[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339366:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000fd3ea555 17ffffc0001001 3:0 ffff9b0360b1cbd0 16937 lru 00000080:00200000:2.0:1769074919.339371:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000c4ef1208 16937: 1< 0 - - > 2< 69373952 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339373:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b1e58bfd 00000080:00200000:2.0:1769074919.339374:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 42. 00000080:00200000:2.0:1769074919.339377:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16938 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339393:0:68465:0:(rw26.c:912:ll_write_end()) pos 69378048, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339397:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000d2bdb8bc[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339400:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000f7d30f5e 17ffffc0001001 3:0 ffff9b0360b1c6c0 16938 lru 00000080:00200000:2.0:1769074919.339405:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000bc66796d 16938: 1< 0 - - > 2< 69378048 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339407:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000d2bdb8bc 00000080:00200000:2.0:1769074919.339408:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 43. 00000080:00200000:2.0:1769074919.339410:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16939 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339425:0:68465:0:(rw26.c:912:ll_write_end()) pos 69382144, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339429:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000e3bcf154[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339432:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000025bf4e21 17ffffc0001001 3:0 ffff9b0360b1c360 16939 lru 00000080:00200000:2.0:1769074919.339437:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000059b05e82 16939: 1< 0 - - > 2< 69382144 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339439:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000e3bcf154 00000080:00200000:2.0:1769074919.339440:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 44. 00000080:00200000:2.0:1769074919.339442:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16940 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339469:0:68465:0:(rw26.c:912:ll_write_end()) pos 69386240, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339474:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000bb2764ad[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339477:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000008560a9ae 17ffffc0001021 2:0 ffff9b0360b1ca20 16940 lru 00000080:00200000:2.0:1769074919.339483:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d907d37a 16940: 1< 0 - - > 2< 69386240 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339485:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000bb2764ad 00000080:00200000:2.0:1769074919.339486:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 45. 00000080:00200000:2.0:1769074919.339488:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16941 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339506:0:68465:0:(rw26.c:912:ll_write_end()) pos 69390336, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339510:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000957fda2e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339513:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000034a891d0 17ffffc0001001 3:0 ffff9b0360b1c1b0 16941 lru 00000080:00200000:2.0:1769074919.339519:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f582ce99 16941: 1< 0 - - > 2< 69390336 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339521:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000957fda2e 00000080:00200000:2.0:1769074919.339522:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 46. 00000080:00200000:2.0:1769074919.339524:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16942 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339544:0:68465:0:(rw26.c:912:ll_write_end()) pos 69394432, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339548:0:68465:0:(rw26.c:969:ll_write_end()) page@000000009bcfff5e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339551:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000e99f56b2 17ffffc0001001 3:0 ffff9b0360b1c798 16942 lru 00000080:00200000:2.0:1769074919.339555:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000edaccc43 16942: 1< 0 - - > 2< 69394432 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339557:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000009bcfff5e 00000080:00200000:2.0:1769074919.339558:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 47. 00000080:00200000:2.0:1769074919.339560:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16943 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339576:0:68465:0:(rw26.c:912:ll_write_end()) pos 69398528, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339580:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000522810d2[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339583:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000094c05130 17ffffc0001001 3:0 ffff9b0360b1caf8 16943 lru 00000080:00200000:2.0:1769074919.339589:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000009f70093f 16943: 1< 0 - - > 2< 69398528 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339591:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000522810d2 00000080:00200000:2.0:1769074919.339592:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 48. 00000080:00200000:2.0:1769074919.339594:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16944 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339611:0:68465:0:(rw26.c:912:ll_write_end()) pos 69402624, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339615:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000c025b399[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339618:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000003720bd88 17ffffc0001001 3:0 ffff9b0360b1cd80 16944 lru 00000080:00200000:2.0:1769074919.339622:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000032ef5676 16944: 1< 0 - - > 2< 69402624 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339625:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000c025b399 00000080:00200000:2.0:1769074919.339626:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 49. 00000080:00200000:2.0:1769074919.339628:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16945 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339656:0:68465:0:(rw26.c:912:ll_write_end()) pos 69406720, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339660:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000ecbb8297[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339663:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000e6e35bc6 17ffffc0001001 3:0 ffff9b0360b1ce58 16945 lru 00000080:00200000:2.0:1769074919.339667:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000005885f104 16945: 1< 0 - - > 2< 69406720 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339669:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000ecbb8297 00000080:00200000:2.0:1769074919.339671:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 50. 00000080:00200000:2.0:1769074919.339673:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16946 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339688:0:68465:0:(rw26.c:912:ll_write_end()) pos 69410816, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339692:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004bdace08[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339694:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000e7fc4bd6 17ffffc0001001 3:0 ffff9b0360b1c438 16946 lru 00000080:00200000:2.0:1769074919.339699:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000cd0e788f 16946: 1< 0 - - > 2< 69410816 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339702:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004bdace08 00000080:00200000:2.0:1769074919.339703:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 51. 00000080:00200000:2.0:1769074919.339705:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16947 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339721:0:68465:0:(rw26.c:912:ll_write_end()) pos 69414912, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339725:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000723ef76e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339728:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000095e1e7de 17ffffc0001001 3:0 ffff9b0360b1c510 16947 lru 00000080:00200000:2.0:1769074919.339733:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000023027372 16947: 1< 0 - - > 2< 69414912 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339735:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000723ef76e 00000080:00200000:2.0:1769074919.339736:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 52. 00000080:00200000:2.0:1769074919.339738:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16948 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339756:0:68465:0:(rw26.c:912:ll_write_end()) pos 69419008, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339760:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000fa10892d[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339763:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000000397ead5 17ffffc0001001 3:0 ffff9b0360b1c948 16948 lru 00000080:00200000:2.0:1769074919.339769:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000029045e23 16948: 1< 0 - - > 2< 69419008 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339771:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000fa10892d 00000080:00200000:2.0:1769074919.339772:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 53. 00000080:00200000:2.0:1769074919.339774:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16949 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339795:0:68465:0:(rw26.c:912:ll_write_end()) pos 69423104, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339799:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004e6f7259[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339801:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000c2920c44 17ffffc0001001 3:0 ffff9b0360b1c000 16949 lru 00000080:00200000:2.0:1769074919.339806:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000003dd9db98 16949: 1< 0 - - > 2< 69423104 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339808:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004e6f7259 00000080:00200000:2.0:1769074919.339810:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 54. 00000080:00200000:2.0:1769074919.339812:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16950 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339829:0:68465:0:(rw26.c:912:ll_write_end()) pos 69427200, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339833:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000f766da6d[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339837:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000059ff5573 17ffffc0001001 3:0 ffff9b0360b80a20 16950 lru 00000080:00200000:2.0:1769074919.339842:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000debb8a6e 16950: 1< 0 - - > 2< 69427200 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339844:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000f766da6d 00000080:00200000:2.0:1769074919.339845:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 55. 00000080:00200000:2.0:1769074919.339848:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16951 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339863:0:68465:0:(rw26.c:912:ll_write_end()) pos 69431296, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339868:0:68465:0:(rw26.c:969:ll_write_end()) page@000000008b182f0b[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339871:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b8630bb9 17ffffc0001001 3:0 ffff9b0360b801b0 16951 lru 00000080:00200000:2.0:1769074919.339876:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000bbf5703f 16951: 1< 0 - - > 2< 69431296 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339879:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000008b182f0b 00000080:00200000:2.0:1769074919.339880:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 56. 00000080:00200000:2.0:1769074919.339882:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16952 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339897:0:68465:0:(rw26.c:912:ll_write_end()) pos 69435392, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339902:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a8144d49[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339905:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000054e8b855 17ffffc0001001 3:0 ffff9b0360b80798 16952 lru 00000080:00200000:2.0:1769074919.339910:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000006be6519e 16952: 1< 0 - - > 2< 69435392 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339913:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a8144d49 00000080:00200000:2.0:1769074919.339914:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 57. 00000080:00200000:2.0:1769074919.339916:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16953 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339930:0:68465:0:(rw26.c:912:ll_write_end()) pos 69439488, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339934:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000e19e7cfc[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339937:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000006b73a93 17ffffc0001001 3:0 ffff9b0360b80af8 16953 lru 00000080:00200000:2.0:1769074919.339943:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e71e7d32 16953: 1< 0 - - > 2< 69439488 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339945:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000e19e7cfc 00000080:00200000:2.0:1769074919.339946:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 58. 00000080:00200000:2.0:1769074919.339948:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16954 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339962:0:68465:0:(rw26.c:912:ll_write_end()) pos 69443584, len 4096, copied 4096 00000080:00200000:2.0:1769074919.339966:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004f8b19f4[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.339969:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000007eb84a02 17ffffc0001001 3:0 ffff9b0360b80d80 16954 lru 00000080:00200000:2.0:1769074919.339974:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f394b42e 16954: 1< 0 - - > 2< 69443584 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.339976:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004f8b19f4 00000080:00200000:2.0:1769074919.339978:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 59. 00000080:00200000:2.0:1769074919.339979:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16955 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.339999:0:68465:0:(rw26.c:912:ll_write_end()) pos 69447680, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340003:0:68465:0:(rw26.c:969:ll_write_end()) page@000000003de9e5aa[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340007:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000062149377 17ffffc0001021 2:0 ffff9b0360b80e58 16955 lru 00000080:00200000:2.0:1769074919.340012:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000003dc7a636 16955: 1< 0 - - > 2< 69447680 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340015:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000003de9e5aa 00000080:00200000:2.0:1769074919.340016:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 60. 00000080:00200000:2.0:1769074919.340017:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16956 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340033:0:68465:0:(rw26.c:912:ll_write_end()) pos 69451776, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340038:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000ef178d79[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340041:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000005878568b 17ffffc0001001 3:0 ffff9b0360b80438 16956 lru 00000080:00200000:2.0:1769074919.340047:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000000c0c31a5 16956: 1< 0 - - > 2< 69451776 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340049:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000ef178d79 00000080:00200000:2.0:1769074919.340050:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 61. 00000080:00200000:2.0:1769074919.340052:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16957 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340068:0:68465:0:(rw26.c:912:ll_write_end()) pos 69455872, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340072:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000677bdafc[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340075:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000017eec55a 17ffffc0001001 3:0 ffff9b0360b80510 16957 lru 00000080:00200000:2.0:1769074919.340081:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000adbd7d02 16957: 1< 0 - - > 2< 69455872 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340083:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000677bdafc 00000080:00200000:2.0:1769074919.340084:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 62. 00000080:00200000:2.0:1769074919.340086:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16958 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340100:0:68465:0:(rw26.c:912:ll_write_end()) pos 69459968, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340105:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000087b1a55a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340108:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000038371186 17ffffc0001001 3:0 ffff9b0360b80948 16958 lru 00000080:00200000:2.0:1769074919.340113:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000004523764 16958: 1< 0 - - > 2< 69459968 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340116:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000087b1a55a 00000080:00200000:2.0:1769074919.340118:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 63. 00000080:00200000:2.0:1769074919.340120:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16959 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340140:0:68465:0:(rw26.c:912:ll_write_end()) pos 69464064, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340145:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b36b91b1[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340147:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000027926686 17ffffc0001001 3:0 ffff9b0360b80000 16959 lru 00000080:00200000:2.0:1769074919.340153:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000044edab1b 16959: 1< 0 - - > 2< 69464064 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340156:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b36b91b1 00000080:00200000:2.0:1769074919.340157:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 64. 00000080:00200000:2.0:1769074919.340160:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16960 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340184:0:68465:0:(rw26.c:912:ll_write_end()) pos 69468160, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340189:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000002b16845[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340192:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000958ab3af 17ffffc0001001 3:0 ffff9b0360b80ca8 16960 lru 00000080:00200000:2.0:1769074919.340197:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000009933d2b0 16960: 1< 0 - - > 2< 69468160 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340200:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000002b16845 00000080:00200000:2.0:1769074919.340201:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 65. 00000080:00200000:2.0:1769074919.340204:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16961 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340219:0:68465:0:(rw26.c:912:ll_write_end()) pos 69472256, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340223:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000c803a7c8[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340226:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000850adc62 17ffffc0001001 3:0 ffff9b0360b805e8 16961 lru 00000080:00200000:2.0:1769074919.340232:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d5afccb2 16961: 1< 0 - - > 2< 69472256 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340269:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000c803a7c8 00000080:00200000:2.0:1769074919.340271:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 66. 00000080:00200000:2.0:1769074919.340273:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16962 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340290:0:68465:0:(rw26.c:912:ll_write_end()) pos 69476352, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340295:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000d2b55311[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340297:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000e2c9eabf 17ffffc0001001 3:0 ffff9b0360b800d8 16962 lru 00000080:00200000:2.0:1769074919.340303:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d4ae2cb6 16962: 1< 0 - - > 2< 69476352 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340305:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000d2b55311 00000080:00200000:2.0:1769074919.340306:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 67. 00000080:00200000:2.0:1769074919.340309:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16963 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340326:0:68465:0:(rw26.c:912:ll_write_end()) pos 69480448, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340330:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000d25eb6dc[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340333:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000009b9d692a 17ffffc0001001 3:0 ffff9b0360b80288 16963 lru 00000080:00200000:2.0:1769074919.340339:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d230ee68 16963: 1< 0 - - > 2< 69480448 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340342:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000d25eb6dc 00000080:00200000:2.0:1769074919.340343:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 68. 00000080:00200000:2.0:1769074919.340345:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16964 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340360:0:68465:0:(rw26.c:912:ll_write_end()) pos 69484544, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340364:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000bfdc71a7[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340367:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000ca4dccfd 17ffffc0001001 3:0 ffff9b0360b80870 16964 lru 00000080:00200000:2.0:1769074919.340373:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000007658a141 16964: 1< 0 - - > 2< 69484544 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340375:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000bfdc71a7 00000080:00200000:2.0:1769074919.340377:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 69. 00000080:00200000:2.0:1769074919.340379:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16965 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340394:0:68465:0:(rw26.c:912:ll_write_end()) pos 69488640, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340398:0:68465:0:(rw26.c:969:ll_write_end()) page@000000005ecb7d55[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340401:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000008d5f9e80 17ffffc0001001 3:0 ffff9b0360b80bd0 16965 lru 00000080:00200000:2.0:1769074919.340407:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000a0afc440 16965: 1< 0 - - > 2< 69488640 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340410:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000005ecb7d55 00000080:00200000:2.0:1769074919.340411:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 70. 00000080:00200000:2.0:1769074919.340413:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16966 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340426:0:68465:0:(rw26.c:912:ll_write_end()) pos 69492736, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340430:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000f64ae9c0[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340433:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000dbe22a83 17ffffc0001001 3:0 ffff9b0360b806c0 16966 lru 00000080:00200000:2.0:1769074919.340439:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000cc100b55 16966: 1< 0 - - > 2< 69492736 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340442:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000f64ae9c0 00000080:00200000:2.0:1769074919.340443:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 71. 00000080:00200000:2.0:1769074919.340445:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16967 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340460:0:68465:0:(rw26.c:912:ll_write_end()) pos 69496832, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340465:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000babfcdd7[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340468:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000dfc1d13d 17ffffc0001001 3:0 ffff9b0360b80360 16967 lru 00000080:00200000:2.0:1769074919.340473:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000a70d702c 16967: 1< 0 - - > 2< 69496832 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340476:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000babfcdd7 00000080:00200000:2.0:1769074919.340477:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 72. 00000080:00200000:2.0:1769074919.340480:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16968 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340496:0:68465:0:(rw26.c:912:ll_write_end()) pos 69500928, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340501:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000e8861744[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340504:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000001f64d2e3 17ffffc0001001 3:0 ffff9b03482fb510 16968 lru 00000080:00200000:2.0:1769074919.340510:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e9a044da 16968: 1< 0 - - > 2< 69500928 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340513:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000e8861744 00000080:00200000:2.0:1769074919.340514:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 73. 00000080:00200000:2.0:1769074919.340516:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16969 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340532:0:68465:0:(rw26.c:912:ll_write_end()) pos 69505024, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340536:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000abb63037[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340539:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000001bac8d52 17ffffc0001001 3:0 ffff9b03482fb948 16969 lru 00000080:00200000:2.0:1769074919.340545:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000034327a76 16969: 1< 0 - - > 2< 69505024 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340547:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000abb63037 00000080:00200000:2.0:1769074919.340549:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 74. 00000080:00200000:2.0:1769074919.340551:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16970 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340572:0:68465:0:(rw26.c:912:ll_write_end()) pos 69509120, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340577:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b9045d41[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340580:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000013f03789 17ffffc0001021 2:0 ffff9b03482fb000 16970 lru 00000080:00200000:2.0:1769074919.340586:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000004f24e0b8 16970: 1< 0 - - > 2< 69509120 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340589:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b9045d41 00000080:00200000:2.0:1769074919.340590:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 75. 00000080:00200000:2.0:1769074919.340592:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16971 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.340606:0:68465:0:(rw26.c:912:ll_write_end()) pos 69513216, len 4096, copied 4096 00000080:00200000:2.0:1769074919.340611:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000daffbfb8[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.340614:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000cff737f4 17ffffc0001001 3:0 ffff9b03482fbca8 16971 lru 00000080:00200000:2.0:1769074919.340618:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000804b04c5 16971: 1< 0 - - > 2< 69513216 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.340621:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000daffbfb8 00000080:00200000:2.0:1769074919.340622:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 76. 00000080:00200000:2.0:1769074919.340624:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16972 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.342774:0:68465:0:(rw26.c:912:ll_write_end()) pos 69517312, len 4096, copied 4096 00000080:00200000:2.0:1769074919.342797:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a8cb67aa[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.342801:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000091d671bf 17ffffc0001001 3:0 ffff9b03482fb5e8 16972 lru 00000080:00200000:2.0:1769074919.342808:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000779652df 16972: 1< 0 - - > 2< 69517312 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.342811:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a8cb67aa 00000080:00200000:2.0:1769074919.342813:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 77. 00000080:00200000:2.0:1769074919.342818:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16973 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.342874:0:68465:0:(rw26.c:912:ll_write_end()) pos 69521408, len 4096, copied 4096 00000080:00200000:2.0:1769074919.342878:0:68465:0:(rw26.c:969:ll_write_end()) page@000000002c616bd7[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.342880:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000df73a3b6 17ffffc0001001 3:0 ffff9b03482fb0d8 16973 lru 00000080:00200000:2.0:1769074919.342885:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d3aea018 16973: 1< 0 - - > 2< 69521408 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.342887:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000002c616bd7 00000080:00200000:2.0:1769074919.342889:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 78. 00000080:00200000:2.0:1769074919.342891:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16974 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.342911:0:68465:0:(rw26.c:912:ll_write_end()) pos 69525504, len 4096, copied 4096 00000080:00200000:2.0:1769074919.342914:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000005cf4c3f[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.342916:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000fce3946a 17ffffc0001001 3:0 ffff9b03482fb288 16974 lru 00000080:00200000:2.0:1769074919.342921:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000058112d1a 16974: 1< 0 - - > 2< 69525504 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.342923:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000005cf4c3f 00000080:00200000:2.0:1769074919.342924:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 79. 00000080:00200000:2.0:1769074919.342925:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16975 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.342941:0:68465:0:(rw26.c:912:ll_write_end()) pos 69529600, len 4096, copied 4096 00000080:00200000:2.0:1769074919.342945:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000059efb462[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.342947:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000008e1895cf 17ffffc0001001 3:0 ffff9b03482fb870 16975 lru 00000080:00200000:2.0:1769074919.342952:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000012a66503 16975: 1< 0 - - > 2< 69529600 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.342953:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000059efb462 00000080:00200000:2.0:1769074919.342954:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 80. 00000080:00200000:2.0:1769074919.342956:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16976 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.342970:0:68465:0:(rw26.c:912:ll_write_end()) pos 69533696, len 4096, copied 4096 00000080:00200000:2.0:1769074919.342973:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000927aad64[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.342976:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000009c950422 17ffffc0001001 3:0 ffff9b03482fbbd0 16976 lru 00000080:00200000:2.0:1769074919.342981:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000009130fe29 16976: 1< 0 - - > 2< 69533696 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.342984:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000927aad64 00000080:00200000:2.0:1769074919.342985:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 81. 00000080:00200000:2.0:1769074919.342986:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16977 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343004:0:68465:0:(rw26.c:912:ll_write_end()) pos 69537792, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343007:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000bda30621[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343010:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000914dbc54 17ffffc0001001 3:0 ffff9b03482fb6c0 16977 lru 00000080:00200000:2.0:1769074919.343015:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000be2f5f97 16977: 1< 0 - - > 2< 69537792 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343018:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000bda30621 00000080:00200000:2.0:1769074919.343019:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 82. 00000080:00200000:2.0:1769074919.343021:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16978 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343040:0:68465:0:(rw26.c:912:ll_write_end()) pos 69541888, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343043:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000c8060030[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343047:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000785d9b91 17ffffc0001001 3:0 ffff9b03482fb360 16978 lru 00000080:00200000:2.0:1769074919.343051:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000056f92078 16978: 1< 0 - - > 2< 69541888 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343054:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000c8060030 00000080:00200000:2.0:1769074919.343055:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 83. 00000080:00200000:2.0:1769074919.343057:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16979 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343073:0:68465:0:(rw26.c:912:ll_write_end()) pos 69545984, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343077:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000eec495cb[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343079:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000bf84c36b 17ffffc0001001 3:0 ffff9b03482fba20 16979 lru 00000080:00200000:2.0:1769074919.343084:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000073ae920 16979: 1< 0 - - > 2< 69545984 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343086:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000eec495cb 00000080:00200000:2.0:1769074919.343087:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 84. 00000080:00200000:2.0:1769074919.343089:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16980 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343103:0:68465:0:(rw26.c:912:ll_write_end()) pos 69550080, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343107:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000cedd6f8c[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343109:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000001b0dc953 17ffffc0001001 3:0 ffff9b03482fb1b0 16980 lru 00000080:00200000:2.0:1769074919.343114:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d886f61c 16980: 1< 0 - - > 2< 69550080 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343116:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000cedd6f8c 00000080:00200000:2.0:1769074919.343117:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 85. 00000080:00200000:2.0:1769074919.343119:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16981 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343133:0:68465:0:(rw26.c:912:ll_write_end()) pos 69554176, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343138:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000029c75729[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343141:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000065ed616e 17ffffc0001001 3:0 ffff9b03482fb798 16981 lru 00000080:00200000:2.0:1769074919.343146:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f327516a 16981: 1< 0 - - > 2< 69554176 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343148:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000029c75729 00000080:00200000:2.0:1769074919.343150:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 86. 00000080:00200000:2.0:1769074919.343152:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16982 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343169:0:68465:0:(rw26.c:912:ll_write_end()) pos 69558272, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343172:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000ce4a33cc[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343175:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000009df0974e 17ffffc0001001 3:0 ffff9b03482fbaf8 16982 lru 00000080:00200000:2.0:1769074919.343180:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000002ee85730 16982: 1< 0 - - > 2< 69558272 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343182:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000ce4a33cc 00000080:00200000:2.0:1769074919.343184:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 87. 00000080:00200000:2.0:1769074919.343186:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16983 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343201:0:68465:0:(rw26.c:912:ll_write_end()) pos 69562368, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343205:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000091ad7fb4[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343207:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b8612709 17ffffc0001001 3:0 ffff9b03482fbd80 16983 lru 00000080:00200000:2.0:1769074919.343212:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008facf0b1 16983: 1< 0 - - > 2< 69562368 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343214:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000091ad7fb4 00000080:00200000:2.0:1769074919.343216:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 88. 00000080:00200000:2.0:1769074919.343217:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16984 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343304:0:68465:0:(rw26.c:912:ll_write_end()) pos 69566464, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343310:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000e253cb1a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343313:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000e3456520 17ffffc0001001 3:0 ffff9b03482fbe58 16984 lru 00000080:00200000:2.0:1769074919.343318:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e449ccce 16984: 1< 0 - - > 2< 69566464 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343320:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000e253cb1a 00000080:00200000:2.0:1769074919.343321:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 89. 00000080:00200000:2.0:1769074919.343323:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16985 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343351:0:68465:0:(rw26.c:912:ll_write_end()) pos 69570560, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343355:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000996fc1d5[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343358:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000f4c9f7cc 17ffffc0001021 2:0 ffff9b03482fb438 16985 lru 00000080:00200000:2.0:1769074919.343364:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000004a1dc76e 16985: 1< 0 - - > 2< 69570560 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343366:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000996fc1d5 00000080:00200000:2.0:1769074919.343367:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 90. 00000080:00200000:2.0:1769074919.343369:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16986 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343390:0:68465:0:(rw26.c:912:ll_write_end()) pos 69574656, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343395:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000426ba258[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343397:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000de09962b 17ffffc0001001 3:0 ffff9b0348360e58 16986 lru 00000080:00200000:2.0:1769074919.343403:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000000649e159 16986: 1< 0 - - > 2< 69574656 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343405:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000426ba258 00000080:00200000:2.0:1769074919.343406:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 91. 00000080:00200000:2.0:1769074919.343409:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16987 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343423:0:68465:0:(rw26.c:912:ll_write_end()) pos 69578752, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343428:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006439d4cb[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343430:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000bf97a61b 17ffffc0001001 3:0 ffff9b0348360438 16987 lru 00000080:00200000:2.0:1769074919.343436:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000061fb815a 16987: 1< 0 - - > 2< 69578752 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343438:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006439d4cb 00000080:00200000:2.0:1769074919.343439:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 92. 00000080:00200000:2.0:1769074919.343441:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16988 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343459:0:68465:0:(rw26.c:912:ll_write_end()) pos 69582848, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343463:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004ef387d7[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343465:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d570310e 17ffffc0001001 3:0 ffff9b0348360510 16988 lru 00000080:00200000:2.0:1769074919.343470:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000003da7a02c 16988: 1< 0 - - > 2< 69582848 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343472:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004ef387d7 00000080:00200000:2.0:1769074919.343474:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 93. 00000080:00200000:2.0:1769074919.343476:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16989 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343492:0:68465:0:(rw26.c:912:ll_write_end()) pos 69586944, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343495:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000098e0b4e7[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343498:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000451aafb0 17ffffc0001001 3:0 ffff9b0348360948 16989 lru 00000080:00200000:2.0:1769074919.343503:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000cfbfff85 16989: 1< 0 - - > 2< 69586944 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343506:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000098e0b4e7 00000080:00200000:2.0:1769074919.343507:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 94. 00000080:00200000:2.0:1769074919.343509:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16990 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343606:0:68465:0:(rw26.c:912:ll_write_end()) pos 69591040, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343612:0:68465:0:(rw26.c:969:ll_write_end()) page@000000001a47d082[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343615:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000005e0c071 17ffffc0001001 3:0 ffff9b0348360000 16990 lru 00000080:00200000:2.0:1769074919.343620:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000daa87db9 16990: 1< 0 - - > 2< 69591040 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343623:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000001a47d082 00000080:00200000:2.0:1769074919.343624:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 95. 00000080:00200000:2.0:1769074919.343626:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16991 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343656:0:68465:0:(rw26.c:912:ll_write_end()) pos 69595136, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343660:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b7eb998b[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343663:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d19bce50 17ffffc0001001 3:0 ffff9b0348360ca8 16991 lru 00000080:00200000:2.0:1769074919.343669:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000594cb6ad 16991: 1< 0 - - > 2< 69595136 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343671:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b7eb998b 00000080:00200000:2.0:1769074919.343673:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 96. 00000080:00200000:2.0:1769074919.343675:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16992 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343693:0:68465:0:(rw26.c:912:ll_write_end()) pos 69599232, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343697:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000ffdfdbed[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343700:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000006d857c01 17ffffc0001001 3:0 ffff9b03483605e8 16992 lru 00000080:00200000:2.0:1769074919.343705:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000009612c2b6 16992: 1< 0 - - > 2< 69599232 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343707:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000ffdfdbed 00000080:00200000:2.0:1769074919.343708:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 97. 00000080:00200000:2.0:1769074919.343710:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16993 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343726:0:68465:0:(rw26.c:912:ll_write_end()) pos 69603328, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343730:0:68465:0:(rw26.c:969:ll_write_end()) page@000000008feab47b[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343733:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000f19a5dab 17ffffc0001001 3:0 ffff9b03483600d8 16993 lru 00000080:00200000:2.0:1769074919.343738:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000558f267f 16993: 1< 0 - - > 2< 69603328 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343740:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000008feab47b 00000080:00200000:2.0:1769074919.343742:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 98. 00000080:00200000:2.0:1769074919.343744:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16994 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343760:0:68465:0:(rw26.c:912:ll_write_end()) pos 69607424, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343764:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000e1c3f952[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343767:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000006663f4cb 17ffffc0001001 3:0 ffff9b0348360288 16994 lru 00000080:00200000:2.0:1769074919.343772:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000000237ddf 16994: 1< 0 - - > 2< 69607424 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343775:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000e1c3f952 00000080:00200000:2.0:1769074919.343776:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 99. 00000080:00200000:2.0:1769074919.343778:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16995 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343794:0:68465:0:(rw26.c:912:ll_write_end()) pos 69611520, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343798:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000100ec70b[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343801:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000aaf2a38a 17ffffc0001001 3:0 ffff9b0348360870 16995 lru 00000080:00200000:2.0:1769074919.343806:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000133a3fbc 16995: 1< 0 - - > 2< 69611520 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343808:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000100ec70b 00000080:00200000:2.0:1769074919.343809:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 100. 00000080:00200000:2.0:1769074919.343812:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16996 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343828:0:68465:0:(rw26.c:912:ll_write_end()) pos 69615616, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343832:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004fc7fdfc[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343835:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000008521d38e 17ffffc0001001 3:0 ffff9b0348360bd0 16996 lru 00000080:00200000:2.0:1769074919.343840:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f1b9d527 16996: 1< 0 - - > 2< 69615616 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343842:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004fc7fdfc 00000080:00200000:2.0:1769074919.343843:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 101. 00000080:00200000:2.0:1769074919.343845:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16997 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343859:0:68465:0:(rw26.c:912:ll_write_end()) pos 69619712, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343863:0:68465:0:(rw26.c:969:ll_write_end()) page@000000005d9520ed[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343866:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000796bc832 17ffffc0001001 3:0 ffff9b03483606c0 16997 lru 00000080:00200000:2.0:1769074919.343871:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000002a1d4d60 16997: 1< 0 - - > 2< 69619712 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343873:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000005d9520ed 00000080:00200000:2.0:1769074919.343874:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 102. 00000080:00200000:2.0:1769074919.343876:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16998 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343893:0:68465:0:(rw26.c:912:ll_write_end()) pos 69623808, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343897:0:68465:0:(rw26.c:969:ll_write_end()) page@000000003449217e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343900:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000c6955b83 17ffffc0001001 3:0 ffff9b0348360360 16998 lru 00000080:00200000:2.0:1769074919.343905:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008b5bd5bc 16998: 1< 0 - - > 2< 69623808 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343908:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000003449217e 00000080:00200000:2.0:1769074919.343909:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 103. 00000080:00200000:2.0:1769074919.343910:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 16999 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343924:0:68465:0:(rw26.c:912:ll_write_end()) pos 69627904, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343928:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000acc1aa06[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343931:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000f581639d 17ffffc0001001 3:0 ffff9b0348360a20 16999 lru 00000080:00200000:2.0:1769074919.343936:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000062df62de 16999: 1< 0 - - > 2< 69627904 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343939:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000acc1aa06 00000080:00200000:2.0:1769074919.343940:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 104. 00000080:00200000:2.0:1769074919.343942:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17000 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343962:0:68465:0:(rw26.c:912:ll_write_end()) pos 69632000, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343967:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000728d90a4[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.343970:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000576c7fb0 17ffffc0001021 2:0 ffff9b03483601b0 17000 lru 00000080:00200000:2.0:1769074919.343976:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000001b422131 17000: 1< 0 - - > 2< 69632000 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.343978:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000728d90a4 00000080:00200000:2.0:1769074919.343980:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 105. 00000080:00200000:2.0:1769074919.343982:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17001 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.343994:0:68465:0:(rw26.c:912:ll_write_end()) pos 69636096, len 4096, copied 4096 00000080:00200000:2.0:1769074919.343998:0:68465:0:(rw26.c:969:ll_write_end()) page@000000008919bf1d[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.344001:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000005fb8e107 17ffffc0001001 3:0 ffff9b0348360798 17001 lru 00000080:00200000:2.0:1769074919.344005:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000009be04df0 17001: 1< 0 - - > 2< 69636096 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.344007:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000008919bf1d 00000080:00200000:2.0:1769074919.344009:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 106. 00000080:00200000:2.0:1769074919.344011:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17002 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.344027:0:68465:0:(rw26.c:912:ll_write_end()) pos 69640192, len 4096, copied 4096 00000080:00200000:2.0:1769074919.344030:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000017db8475[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.344033:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000009dc94d7b 17ffffc0001001 3:0 ffff9b0348360af8 17002 lru 00000080:00200000:2.0:1769074919.344037:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e31dd676 17002: 1< 0 - - > 2< 69640192 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.344039:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000017db8475 00000080:00200000:2.0:1769074919.344041:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 107. 00000080:00200000:2.0:1769074919.344043:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17003 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.344058:0:68465:0:(rw26.c:912:ll_write_end()) pos 69644288, len 4096, copied 4096 00000080:00200000:2.0:1769074919.344061:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000065ce488a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.344064:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000000b754b5c 17ffffc0001001 3:0 ffff9b0348360d80 17003 lru 00000080:00200000:2.0:1769074919.344071:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000003d5a5f04 17003: 1< 0 - - > 2< 69644288 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.344073:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000065ce488a 00000080:00200000:2.0:1769074919.344074:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 108. 00000080:00200000:2.0:1769074919.344076:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17004 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.344094:0:68465:0:(rw26.c:912:ll_write_end()) pos 69648384, len 4096, copied 4096 00000080:00200000:2.0:1769074919.344098:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000fcaa2416[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.344101:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000009ee6c473 17ffffc0001001 3:0 ffff9b0346f60288 17004 lru 00000080:00200000:2.0:1769074919.344106:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000b622ebe0 17004: 1< 0 - - > 2< 69648384 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.344109:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000fcaa2416 00000080:00200000:2.0:1769074919.344110:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 109. 00000080:00200000:2.0:1769074919.344112:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17005 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.344131:0:68465:0:(rw26.c:912:ll_write_end()) pos 69652480, len 4096, copied 4096 00000080:00200000:2.0:1769074919.344135:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000087e86de6[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.344138:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000004b4e5b7 17ffffc0001001 3:0 ffff9b0346f60870 17005 lru 00000080:00200000:2.0:1769074919.344143:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000009358c96f 17005: 1< 0 - - > 2< 69652480 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.344146:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000087e86de6 00000080:00200000:2.0:1769074919.344147:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 110. 00000080:00200000:2.0:1769074919.344150:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17006 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.344166:0:68465:0:(rw26.c:912:ll_write_end()) pos 69656576, len 4096, copied 4096 00000080:00200000:2.0:1769074919.344170:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b3d72ced[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.344173:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b4aaaa5b 17ffffc0001001 3:0 ffff9b0346f60bd0 17006 lru 00000080:00200000:2.0:1769074919.344178:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000caf87f9a 17006: 1< 0 - - > 2< 69656576 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.344181:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b3d72ced 00000080:00200000:2.0:1769074919.344182:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 111. 00000080:00200000:2.0:1769074919.344184:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17007 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.344198:0:68465:0:(rw26.c:912:ll_write_end()) pos 69660672, len 4096, copied 4096 00000080:00200000:2.0:1769074919.344203:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000664c9e2c[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.344205:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d7a8e6e0 17ffffc0001001 3:0 ffff9b0346f606c0 17007 lru 00000080:00200000:2.0:1769074919.344210:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000930e8d7f 17007: 1< 0 - - > 2< 69660672 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.344212:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000664c9e2c 00000080:00200000:2.0:1769074919.344213:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 112. 00000080:00200000:2.0:1769074919.344215:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17008 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.344229:0:68465:0:(rw26.c:912:ll_write_end()) pos 69664768, len 4096, copied 4096 00000080:00200000:2.0:1769074919.345737:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000046adcef8[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.345747:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000be32b060 17ffffc0001001 3:0 ffff9b0346f60360 17008 lru 00000080:00200000:2.0:1769074919.345757:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000c23f7486 17008: 1< 0 - - > 2< 69664768 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.345761:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000046adcef8 00000080:00200000:2.0:1769074919.345763:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 113. 00000080:00200000:2.0:1769074919.345770:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17009 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.345830:0:68465:0:(rw26.c:912:ll_write_end()) pos 69668864, len 4096, copied 4096 00000080:00200000:2.0:1769074919.345837:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000cf790e55[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.345841:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000be38ca34 17ffffc0001001 3:0 ffff9b0346f60a20 17009 lru 00000080:00200000:2.0:1769074919.345846:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000002c0618c2 17009: 1< 0 - - > 2< 69668864 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.345849:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000cf790e55 00000080:00200000:2.0:1769074919.345850:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 114. 00000080:00200000:2.0:1769074919.345852:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17010 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.345872:0:68465:0:(rw26.c:912:ll_write_end()) pos 69672960, len 4096, copied 4096 00000080:00200000:2.0:1769074919.345876:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000ff83447d[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.345880:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000a54f4c90 17ffffc0001001 3:0 ffff9b0346f601b0 17010 lru 00000080:00200000:2.0:1769074919.345885:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000a885f056 17010: 1< 0 - - > 2< 69672960 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.345888:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000ff83447d 00000080:00200000:2.0:1769074919.345889:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 115. 00000080:00200000:2.0:1769074919.345891:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17011 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.345906:0:68465:0:(rw26.c:912:ll_write_end()) pos 69677056, len 4096, copied 4096 00000080:00200000:2.0:1769074919.345910:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000132471bd[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.345913:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d3462240 17ffffc0001001 3:0 ffff9b0346f60798 17011 lru 00000080:00200000:2.0:1769074919.345919:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000096ad417f 17011: 1< 0 - - > 2< 69677056 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.345921:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000132471bd 00000080:00200000:2.0:1769074919.345923:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 116. 00000080:00200000:2.0:1769074919.345925:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17012 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.345943:0:68465:0:(rw26.c:912:ll_write_end()) pos 69681152, len 4096, copied 4096 00000080:00200000:2.0:1769074919.345948:0:68465:0:(rw26.c:969:ll_write_end()) page@000000007703a0f4[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.345951:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000a7b3a4a9 17ffffc0001001 3:0 ffff9b0346f60af8 17012 lru 00000080:00200000:2.0:1769074919.345957:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000c82ff6f9 17012: 1< 0 - - > 2< 69681152 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.345959:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000007703a0f4 00000080:00200000:2.0:1769074919.345961:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 117. 00000080:00200000:2.0:1769074919.345963:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17013 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.345978:0:68465:0:(rw26.c:912:ll_write_end()) pos 69685248, len 4096, copied 4096 00000080:00200000:2.0:1769074919.345982:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000bafaabb5[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.345986:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000043b85af6 17ffffc0001001 3:0 ffff9b0346f60d80 17013 lru 00000080:00200000:2.0:1769074919.345992:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000070ff9a98 17013: 1< 0 - - > 2< 69685248 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.345994:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000bafaabb5 00000080:00200000:2.0:1769074919.345995:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 118. 00000080:00200000:2.0:1769074919.345997:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17014 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.346014:0:68465:0:(rw26.c:912:ll_write_end()) pos 69689344, len 4096, copied 4096 00000080:00200000:2.0:1769074919.346018:0:68465:0:(rw26.c:969:ll_write_end()) page@000000000d27bf4b[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.346021:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000000321ab71 17ffffc0001001 3:0 ffff9b0346f60e58 17014 lru 00000080:00200000:2.0:1769074919.346026:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000015590d10 17014: 1< 0 - - > 2< 69689344 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.346029:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000000d27bf4b 00000080:00200000:2.0:1769074919.346030:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 119. 00000080:00200000:2.0:1769074919.346032:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17015 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.346058:0:68465:0:(rw26.c:912:ll_write_end()) pos 69693440, len 4096, copied 4096 00000080:00200000:2.0:1769074919.346063:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000826d48e0[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.346066:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000063f30193 17ffffc0001021 2:0 ffff9b0346f60438 17015 lru 00000080:00200000:2.0:1769074919.346071:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000a6d95524 17015: 1< 0 - - > 2< 69693440 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.346074:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000826d48e0 00000080:00200000:2.0:1769074919.346075:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 120. 00000080:00200000:2.0:1769074919.346077:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17016 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.346093:0:68465:0:(rw26.c:912:ll_write_end()) pos 69697536, len 4096, copied 4096 00000080:00200000:2.0:1769074919.346098:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000001acea65[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.346100:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000086fff7e0 17ffffc0001001 3:0 ffff9b0346f60510 17016 lru 00000080:00200000:2.0:1769074919.346106:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000428525f5 17016: 1< 0 - - > 2< 69697536 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.346108:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000001acea65 00000080:00200000:2.0:1769074919.346110:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 121. 00000080:00200000:2.0:1769074919.346111:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17017 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.346128:0:68465:0:(rw26.c:912:ll_write_end()) pos 69701632, len 4096, copied 4096 00000080:00200000:2.0:1769074919.346133:0:68465:0:(rw26.c:969:ll_write_end()) page@000000002cc9e4ee[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.346136:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000da1cea5b 17ffffc0001001 3:0 ffff9b0346f60948 17017 lru 00000080:00200000:2.0:1769074919.346141:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008add15fe 17017: 1< 0 - - > 2< 69701632 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.346144:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000002cc9e4ee 00000080:00200000:2.0:1769074919.346145:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 122. 00000080:00200000:2.0:1769074919.346147:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17018 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.346164:0:68465:0:(rw26.c:912:ll_write_end()) pos 69705728, len 4096, copied 4096 00000080:00200000:2.0:1769074919.346169:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000033212d95[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.346172:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000004817ba9c 17ffffc0001001 3:0 ffff9b0346f60000 17018 lru 00000080:00200000:2.0:1769074919.346178:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000073fe5a43 17018: 1< 0 - - > 2< 69705728 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.346181:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000033212d95 00000080:00200000:2.0:1769074919.346183:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 123. 00000080:00200000:2.0:1769074919.346185:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17019 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.346202:0:68465:0:(rw26.c:912:ll_write_end()) pos 69709824, len 4096, copied 4096 00000080:00200000:2.0:1769074919.346206:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a110555a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.346210:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000001c4bfdb6 17ffffc0001001 3:0 ffff9b0346f60ca8 17019 lru 00000080:00200000:2.0:1769074919.346216:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008603ca38 17019: 1< 0 - - > 2< 69709824 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.346218:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a110555a 00000080:00200000:2.0:1769074919.346220:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 124. 00000080:00200000:2.0:1769074919.346222:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17020 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.347607:0:68465:0:(rw26.c:912:ll_write_end()) pos 69713920, len 4096, copied 4096 00000080:00200000:2.0:1769074919.347622:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000037d867a6[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.347626:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000057f29c99 17ffffc0001001 3:0 ffff9b0346f605e8 17020 lru 00000080:00200000:2.0:1769074919.347634:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000251dc7e4 17020: 1< 0 - - > 2< 69713920 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.347647:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000037d867a6 00000080:00200000:2.0:1769074919.347649:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 125. 00000080:00200000:2.0:1769074919.347654:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17021 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.347696:0:68465:0:(rw26.c:912:ll_write_end()) pos 69718016, len 4096, copied 4096 00000080:00200000:2.0:1769074919.347701:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000cdeffd56[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.347705:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000e70993aa 17ffffc0001001 3:0 ffff9b0346f600d8 17021 lru 00000080:00200000:2.0:1769074919.347710:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000cd16a500 17021: 1< 0 - - > 2< 69718016 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.347712:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000cdeffd56 00000080:00200000:2.0:1769074919.347714:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 126. 00000080:00200000:2.0:1769074919.347716:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17022 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.347734:0:68465:0:(rw26.c:912:ll_write_end()) pos 69722112, len 4096, copied 4096 00000080:00200000:2.0:1769074919.347739:0:68465:0:(rw26.c:969:ll_write_end()) page@000000005bbffac3[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.347742:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000ec90961f 17ffffc0001001 3:0 ffff9b0353150ca8 17022 lru 00000080:00200000:2.0:1769074919.347747:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000b84bfa14 17022: 1< 0 - - > 2< 69722112 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.347750:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000005bbffac3 00000080:00200000:2.0:1769074919.347751:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 127. 00000080:00200000:2.0:1769074919.347754:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17023 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.347775:0:68465:0:(rw26.c:912:ll_write_end()) pos 69726208, len 4096, copied 4096 00000080:00200000:2.0:1769074919.347780:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000056d437e5[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.347783:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000bff78f0b 17ffffc0001001 3:0 ffff9b03531505e8 17023 lru 00000080:00200000:2.0:1769074919.347789:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000001324607c 17023: 1< 0 - - > 2< 69726208 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.347792:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000056d437e5 00000080:00200000:2.0:1769074919.347793:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 128. 00000080:00200000:2.0:1769074919.347795:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17024 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.347819:0:68465:0:(rw26.c:912:ll_write_end()) pos 69730304, len 4096, copied 4096 00000080:00200000:2.0:1769074919.347824:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000073beba0a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.347827:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d3dc614c 17ffffc0001001 3:0 ffff9b03531500d8 17024 lru 00000080:00200000:2.0:1769074919.347834:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000010aaf8f0 17024: 1< 0 - - > 2< 69730304 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.347836:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000073beba0a 00000080:00200000:2.0:1769074919.347837:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 129. 00000080:00200000:2.0:1769074919.347840:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17025 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.347856:0:68465:0:(rw26.c:912:ll_write_end()) pos 69734400, len 4096, copied 4096 00000080:00200000:2.0:1769074919.347861:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000014e87049[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.347864:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000070b82d29 17ffffc0001001 3:0 ffff9b0353150288 17025 lru 00000080:00200000:2.0:1769074919.347870:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000b3eb6c08 17025: 1< 0 - - > 2< 69734400 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.347873:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000014e87049 00000080:00200000:2.0:1769074919.347874:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 130. 00000080:00200000:2.0:1769074919.347876:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17026 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.347895:0:68465:0:(rw26.c:912:ll_write_end()) pos 69738496, len 4096, copied 4096 00000080:00200000:2.0:1769074919.347900:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006a36bb00[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.347903:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000004eab8bc4 17ffffc0001001 3:0 ffff9b0353150870 17026 lru 00000080:00200000:2.0:1769074919.347908:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000007ad584a9 17026: 1< 0 - - > 2< 69738496 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.347911:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006a36bb00 00000080:00200000:2.0:1769074919.347912:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 131. 00000080:00200000:2.0:1769074919.347914:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17027 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.347930:0:68465:0:(rw26.c:912:ll_write_end()) pos 69742592, len 4096, copied 4096 00000080:00200000:2.0:1769074919.347934:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000acdcdb2b[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.347937:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000850a1f50 17ffffc0001001 3:0 ffff9b0353150bd0 17027 lru 00000080:00200000:2.0:1769074919.347943:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000c045836b 17027: 1< 0 - - > 2< 69742592 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.347945:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000acdcdb2b 00000080:00200000:2.0:1769074919.347947:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 132. 00000080:00200000:2.0:1769074919.347949:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17028 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.347963:0:68465:0:(rw26.c:912:ll_write_end()) pos 69746688, len 4096, copied 4096 00000080:00200000:2.0:1769074919.347968:0:68465:0:(rw26.c:969:ll_write_end()) page@000000001eaca902[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.347970:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000001a7309e7 17ffffc0001001 3:0 ffff9b03531506c0 17028 lru 00000080:00200000:2.0:1769074919.347976:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000cc928ac8 17028: 1< 0 - - > 2< 69746688 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.347978:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000001eaca902 00000080:00200000:2.0:1769074919.347979:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 133. 00000080:00200000:2.0:1769074919.347981:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17029 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.347997:0:68465:0:(rw26.c:912:ll_write_end()) pos 69750784, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348001:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000027e0c6f8[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348004:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000f4bb304d 17ffffc0001001 3:0 ffff9b0353150360 17029 lru 00000080:00200000:2.0:1769074919.348010:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e2e1e29c 17029: 1< 0 - - > 2< 69750784 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348013:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000027e0c6f8 00000080:00200000:2.0:1769074919.348014:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 134. 00000080:00200000:2.0:1769074919.348016:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17030 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348039:0:68465:0:(rw26.c:912:ll_write_end()) pos 69754880, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348043:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000dcde033e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348046:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000084d1657d 17ffffc0001021 2:0 ffff9b0353150a20 17030 lru 00000080:00200000:2.0:1769074919.348052:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000039023094 17030: 1< 0 - - > 2< 69754880 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348054:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000dcde033e 00000080:00200000:2.0:1769074919.348055:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 135. 00000080:00200000:2.0:1769074919.348057:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17031 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348075:0:68465:0:(rw26.c:912:ll_write_end()) pos 69758976, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348080:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000836ac6c6[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348083:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000000966a54f 17ffffc0001001 3:0 ffff9b03531501b0 17031 lru 00000080:00200000:2.0:1769074919.348088:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d0ae3826 17031: 1< 0 - - > 2< 69758976 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348091:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000836ac6c6 00000080:00200000:2.0:1769074919.348092:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 136. 00000080:00200000:2.0:1769074919.348094:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17032 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348110:0:68465:0:(rw26.c:912:ll_write_end()) pos 69763072, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348114:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000e21b04be[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348117:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000000fe21d42 17ffffc0001001 3:0 ffff9b0353150798 17032 lru 00000080:00200000:2.0:1769074919.348121:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008a78b691 17032: 1< 0 - - > 2< 69763072 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348124:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000e21b04be 00000080:00200000:2.0:1769074919.348125:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 137. 00000080:00200000:2.0:1769074919.348127:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17033 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348144:0:68465:0:(rw26.c:912:ll_write_end()) pos 69767168, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348148:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a3ae76b4[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348151:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000004f69a449 17ffffc0001001 3:0 ffff9b0353150af8 17033 lru 00000080:00200000:2.0:1769074919.348155:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000004bef52cb 17033: 1< 0 - - > 2< 69767168 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348157:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a3ae76b4 00000080:00200000:2.0:1769074919.348158:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 138. 00000080:00200000:2.0:1769074919.348160:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17034 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348177:0:68465:0:(rw26.c:912:ll_write_end()) pos 69771264, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348182:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004df6ddad[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348185:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000001c2433ce 17ffffc0001001 3:0 ffff9b0353150d80 17034 lru 00000080:00200000:2.0:1769074919.348189:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d5f584b2 17034: 1< 0 - - > 2< 69771264 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348192:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004df6ddad 00000080:00200000:2.0:1769074919.348193:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 139. 00000080:00200000:2.0:1769074919.348194:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17035 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348208:0:68465:0:(rw26.c:912:ll_write_end()) pos 69775360, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348212:0:68465:0:(rw26.c:969:ll_write_end()) page@000000009d583d2a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348214:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000aa6ef31f 17ffffc0001001 3:0 ffff9b0353150e58 17035 lru 00000080:00200000:2.0:1769074919.348219:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000022894238 17035: 1< 0 - - > 2< 69775360 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348222:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000009d583d2a 00000080:00200000:2.0:1769074919.348223:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 140. 00000080:00200000:2.0:1769074919.348225:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17036 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348449:0:68465:0:(rw26.c:912:ll_write_end()) pos 69779456, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348454:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000ee4c0dc2[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348457:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000e6f0aa0d 17ffffc0001001 3:0 ffff9b0353150438 17036 lru 00000080:00200000:2.0:1769074919.348462:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000000411e5a8 17036: 1< 0 - - > 2< 69779456 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348464:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000ee4c0dc2 00000080:00200000:2.0:1769074919.348465:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 141. 00000080:00200000:2.0:1769074919.348467:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17037 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348484:0:68465:0:(rw26.c:912:ll_write_end()) pos 69783552, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348487:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000037221f08[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348490:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000015f2c978 17ffffc0001001 3:0 ffff9b0353150510 17037 lru 00000080:00200000:2.0:1769074919.348496:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e9c8434b 17037: 1< 0 - - > 2< 69783552 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348498:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000037221f08 00000080:00200000:2.0:1769074919.348499:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 142. 00000080:00200000:2.0:1769074919.348501:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17038 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348579:0:68465:0:(rw26.c:912:ll_write_end()) pos 69787648, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348584:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000397eef0a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348587:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000c7b53afb 17ffffc0001001 3:0 ffff9b0353150948 17038 lru 00000080:00200000:2.0:1769074919.348593:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000c972a3a6 17038: 1< 0 - - > 2< 69787648 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348595:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000397eef0a 00000080:00200000:2.0:1769074919.348596:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 143. 00000080:00200000:2.0:1769074919.348598:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17039 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348614:0:68465:0:(rw26.c:912:ll_write_end()) pos 69791744, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348618:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000f7c495b4[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348621:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000009ce3ae14 17ffffc0001001 3:0 ffff9b0353150000 17039 lru 00000080:00200000:2.0:1769074919.348626:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e6c417ff 17039: 1< 0 - - > 2< 69791744 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348628:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000f7c495b4 00000080:00200000:2.0:1769074919.348629:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 144. 00000080:00200000:2.0:1769074919.348631:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17040 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348660:0:68465:0:(rw26.c:912:ll_write_end()) pos 69795840, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348664:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000186da121[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348667:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000bd3a58c1 17ffffc0001001 3:0 ffff9b0343b070d8 17040 lru 00000080:00200000:2.0:1769074919.348672:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f392a910 17040: 1< 0 - - > 2< 69795840 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348674:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000186da121 00000080:00200000:2.0:1769074919.348675:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 145. 00000080:00200000:2.0:1769074919.348677:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17041 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348692:0:68465:0:(rw26.c:912:ll_write_end()) pos 69799936, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348696:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000916f5622[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348699:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000007244a50c 17ffffc0001001 3:0 ffff9b0343b07288 17041 lru 00000080:00200000:2.0:1769074919.348704:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000006a7056f4 17041: 1< 0 - - > 2< 69799936 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348706:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000916f5622 00000080:00200000:2.0:1769074919.348707:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 146. 00000080:00200000:2.0:1769074919.348709:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17042 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348731:0:68465:0:(rw26.c:912:ll_write_end()) pos 69804032, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348734:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b0e327aa[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348737:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000df16254c 17ffffc0001001 3:0 ffff9b0343b07870 17042 lru 00000080:00200000:2.0:1769074919.348742:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000003446666c 17042: 1< 0 - - > 2< 69804032 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348744:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b0e327aa 00000080:00200000:2.0:1769074919.348745:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 147. 00000080:00200000:2.0:1769074919.348746:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17043 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348761:0:68465:0:(rw26.c:912:ll_write_end()) pos 69808128, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348765:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000018e3561[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348767:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000009f6013b3 17ffffc0001001 3:0 ffff9b0343b07bd0 17043 lru 00000080:00200000:2.0:1769074919.348771:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000ed0610ed 17043: 1< 0 - - > 2< 69808128 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348773:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000018e3561 00000080:00200000:2.0:1769074919.348774:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 148. 00000080:00200000:2.0:1769074919.348776:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17044 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348791:0:68465:0:(rw26.c:912:ll_write_end()) pos 69812224, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348794:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000bc2b6199[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348797:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000002e64373a 17ffffc0001001 3:0 ffff9b0343b076c0 17044 lru 00000080:00200000:2.0:1769074919.348801:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000000d6092b4 17044: 1< 0 - - > 2< 69812224 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348804:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000bc2b6199 00000080:00200000:2.0:1769074919.348805:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 149. 00000080:00200000:2.0:1769074919.348806:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17045 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348826:0:68465:0:(rw26.c:912:ll_write_end()) pos 69816320, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348831:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006c5b2207[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348833:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000044cb962 17ffffc0001021 2:0 ffff9b0343b07360 17045 lru 00000080:00200000:2.0:1769074919.348838:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000649565bb 17045: 1< 0 - - > 2< 69816320 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348840:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006c5b2207 00000080:00200000:2.0:1769074919.348841:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 150. 00000080:00200000:2.0:1769074919.348843:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17046 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348858:0:68465:0:(rw26.c:912:ll_write_end()) pos 69820416, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348862:0:68465:0:(rw26.c:969:ll_write_end()) page@000000000f5b2bfa[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348864:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000007b13bc2f 17ffffc0001001 3:0 ffff9b0343b07a20 17046 lru 00000080:00200000:2.0:1769074919.348869:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000016b7fc50 17046: 1< 0 - - > 2< 69820416 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348871:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000000f5b2bfa 00000080:00200000:2.0:1769074919.348872:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 151. 00000080:00200000:2.0:1769074919.348873:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17047 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348888:0:68465:0:(rw26.c:912:ll_write_end()) pos 69824512, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348892:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b69079d3[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348895:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000ccac90ca 17ffffc0001001 3:0 ffff9b0343b071b0 17047 lru 00000080:00200000:2.0:1769074919.348900:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000001c24ce9d 17047: 1< 0 - - > 2< 69824512 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348902:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b69079d3 00000080:00200000:2.0:1769074919.348903:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 152. 00000080:00200000:2.0:1769074919.348905:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17048 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348917:0:68465:0:(rw26.c:912:ll_write_end()) pos 69828608, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348921:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000c536692f[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348923:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000db7efdb2 17ffffc0001001 3:0 ffff9b0343b07798 17048 lru 00000080:00200000:2.0:1769074919.348928:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000a763a1b0 17048: 1< 0 - - > 2< 69828608 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348930:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000c536692f 00000080:00200000:2.0:1769074919.348931:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 153. 00000080:00200000:2.0:1769074919.348933:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17049 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348947:0:68465:0:(rw26.c:912:ll_write_end()) pos 69832704, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348950:0:68465:0:(rw26.c:969:ll_write_end()) page@000000008b30c35a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348953:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000a83949d8 17ffffc0001001 3:0 ffff9b0343b07af8 17049 lru 00000080:00200000:2.0:1769074919.348958:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000139cb4d2 17049: 1< 0 - - > 2< 69832704 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348961:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000008b30c35a 00000080:00200000:2.0:1769074919.348962:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 154. 00000080:00200000:2.0:1769074919.348963:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17050 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.348980:0:68465:0:(rw26.c:912:ll_write_end()) pos 69836800, len 4096, copied 4096 00000080:00200000:2.0:1769074919.348984:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000fcc3ba7a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.348987:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000001aaff357 17ffffc0001001 3:0 ffff9b0343b07d80 17050 lru 00000080:00200000:2.0:1769074919.348991:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000fcad2f1c 17050: 1< 0 - - > 2< 69836800 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.348994:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000fcc3ba7a 00000080:00200000:2.0:1769074919.348995:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 155. 00000080:00200000:2.0:1769074919.348997:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17051 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349014:0:68465:0:(rw26.c:912:ll_write_end()) pos 69840896, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349018:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000fcdadeae[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349021:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000000d38a942 17ffffc0001001 3:0 ffff9b0343b07e58 17051 lru 00000080:00200000:2.0:1769074919.349025:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000015e664b6 17051: 1< 0 - - > 2< 69840896 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349028:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000fcdadeae 00000080:00200000:2.0:1769074919.349029:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 156. 00000080:00200000:2.0:1769074919.349030:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17052 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349046:0:68465:0:(rw26.c:912:ll_write_end()) pos 69844992, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349050:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000022e922b6[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349053:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000615a3306 17ffffc0001001 3:0 ffff9b0343b07438 17052 lru 00000080:00200000:2.0:1769074919.349058:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000a46b4b70 17052: 1< 0 - - > 2< 69844992 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349061:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000022e922b6 00000080:00200000:2.0:1769074919.349062:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 157. 00000080:00200000:2.0:1769074919.349064:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17053 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349080:0:68465:0:(rw26.c:912:ll_write_end()) pos 69849088, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349085:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000014677963[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349087:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000fa505d41 17ffffc0001001 3:0 ffff9b0343b07510 17053 lru 00000080:00200000:2.0:1769074919.349093:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000078af367a 17053: 1< 0 - - > 2< 69849088 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349096:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000014677963 00000080:00200000:2.0:1769074919.349097:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 158. 00000080:00200000:2.0:1769074919.349099:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17054 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349118:0:68465:0:(rw26.c:912:ll_write_end()) pos 69853184, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349121:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000fea89b00[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349124:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000004268e97c 17ffffc0001001 3:0 ffff9b0343b07948 17054 lru 00000080:00200000:2.0:1769074919.349129:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000005f9803b3 17054: 1< 0 - - > 2< 69853184 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349131:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000fea89b00 00000080:00200000:2.0:1769074919.349132:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 159. 00000080:00200000:2.0:1769074919.349134:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17055 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349152:0:68465:0:(rw26.c:912:ll_write_end()) pos 69857280, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349155:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000df5911ed[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349157:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000001bbad798 17ffffc0001001 3:0 ffff9b0343b07000 17055 lru 00000080:00200000:2.0:1769074919.349162:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008c155160 17055: 1< 0 - - > 2< 69857280 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349164:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000df5911ed 00000080:00200000:2.0:1769074919.349166:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 160. 00000080:00200000:2.0:1769074919.349168:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17056 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349184:0:68465:0:(rw26.c:912:ll_write_end()) pos 69861376, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349188:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000ee47a9e8[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349190:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000006d3d8c8d 17ffffc0001001 3:0 ffff9b0343b07ca8 17056 lru 00000080:00200000:2.0:1769074919.349195:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000016d5986 17056: 1< 0 - - > 2< 69861376 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349197:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000ee47a9e8 00000080:00200000:2.0:1769074919.349198:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 161. 00000080:00200000:2.0:1769074919.349200:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17057 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349213:0:68465:0:(rw26.c:912:ll_write_end()) pos 69865472, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349217:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000058e413b1[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349220:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d4eba66e 17ffffc0001001 3:0 ffff9b0343b075e8 17057 lru 00000080:00200000:2.0:1769074919.349225:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000cd527623 17057: 1< 0 - - > 2< 69865472 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349226:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000058e413b1 00000080:00200000:2.0:1769074919.349228:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 162. 00000080:00200000:2.0:1769074919.349229:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17058 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349282:0:68465:0:(rw26.c:912:ll_write_end()) pos 69869568, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349287:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000d50dd174[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349289:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000057376f57 17ffffc0001001 3:0 ffff9b03432fc438 17058 lru 00000080:00200000:2.0:1769074919.349294:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000561ae459 17058: 1< 0 - - > 2< 69869568 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349296:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000d50dd174 00000080:00200000:2.0:1769074919.349298:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 163. 00000080:00200000:2.0:1769074919.349300:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17059 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349316:0:68465:0:(rw26.c:912:ll_write_end()) pos 69873664, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349320:0:68465:0:(rw26.c:969:ll_write_end()) page@000000009c9fbd20[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349322:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000f404cb77 17ffffc0001001 3:0 ffff9b03432fce58 17059 lru 00000080:00200000:2.0:1769074919.349326:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000004aa9272d 17059: 1< 0 - - > 2< 69873664 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349329:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000009c9fbd20 00000080:00200000:2.0:1769074919.349330:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 164. 00000080:00200000:2.0:1769074919.349332:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17060 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349350:0:68465:0:(rw26.c:912:ll_write_end()) pos 69877760, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349355:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000016592a4f[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349358:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000000602a893 17ffffc0001021 2:0 ffff9b03432fcd80 17060 lru 00000080:00200000:2.0:1769074919.349364:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000bca666c4 17060: 1< 0 - - > 2< 69877760 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349366:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000016592a4f 00000080:00200000:2.0:1769074919.349367:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 165. 00000080:00200000:2.0:1769074919.349369:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17061 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349387:0:68465:0:(rw26.c:912:ll_write_end()) pos 69881856, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349392:0:68465:0:(rw26.c:969:ll_write_end()) page@000000001231dc59[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349394:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000001951f217 17ffffc0001001 3:0 ffff9b03432fcaf8 17061 lru 00000080:00200000:2.0:1769074919.349399:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000c005cce7 17061: 1< 0 - - > 2< 69881856 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349402:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000001231dc59 00000080:00200000:2.0:1769074919.349403:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 166. 00000080:00200000:2.0:1769074919.349405:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17062 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349419:0:68465:0:(rw26.c:912:ll_write_end()) pos 69885952, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349423:0:68465:0:(rw26.c:969:ll_write_end()) page@000000000be4d044[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349426:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000baae5b43 17ffffc0001001 3:0 ffff9b03432fc798 17062 lru 00000080:00200000:2.0:1769074919.349431:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000006817939d 17062: 1< 0 - - > 2< 69885952 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349434:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000000be4d044 00000080:00200000:2.0:1769074919.349435:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 167. 00000080:00200000:2.0:1769074919.349437:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17063 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349453:0:68465:0:(rw26.c:912:ll_write_end()) pos 69890048, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349458:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000d18609a8[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349461:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000000c182bec 17ffffc0001001 3:0 ffff9b03432fc1b0 17063 lru 00000080:00200000:2.0:1769074919.349467:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000005730e2a5 17063: 1< 0 - - > 2< 69890048 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349469:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000d18609a8 00000080:00200000:2.0:1769074919.349471:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 168. 00000080:00200000:2.0:1769074919.349473:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17064 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.349486:0:68465:0:(rw26.c:912:ll_write_end()) pos 69894144, len 4096, copied 4096 00000080:00200000:2.0:1769074919.349490:0:68465:0:(rw26.c:969:ll_write_end()) page@000000003f824984[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.349492:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000386d8f79 17ffffc0001001 3:0 ffff9b03432fca20 17064 lru 00000080:00200000:2.0:1769074919.349498:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008b592f14 17064: 1< 0 - - > 2< 69894144 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.349500:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000003f824984 00000080:00200000:2.0:1769074919.349501:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 169. 00000080:00200000:2.0:1769074919.349503:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17065 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.354726:0:68465:0:(rw26.c:912:ll_write_end()) pos 69898240, len 4096, copied 4096 00000080:00200000:2.0:1769074919.354756:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006d8ecac4[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.354761:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000002f4cb450 17ffffc0001001 3:0 ffff9b03432fc360 17065 lru 00000080:00200000:2.0:1769074919.354771:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000a3414afb 17065: 1< 0 - - > 2< 69898240 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.354774:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006d8ecac4 00000080:00200000:2.0:1769074919.354776:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 170. 00000080:00200000:2.0:1769074919.354783:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17066 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.354867:0:68465:0:(rw26.c:912:ll_write_end()) pos 69902336, len 4096, copied 4096 00000080:00200000:2.0:1769074919.354872:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000be714b87[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.354876:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000000fed78f 17ffffc0001001 3:0 ffff9b03432fc6c0 17066 lru 00000080:00200000:2.0:1769074919.354882:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000050722255 17066: 1< 0 - - > 2< 69902336 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.354885:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000be714b87 00000080:00200000:2.0:1769074919.354886:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 171. 00000080:00200000:2.0:1769074919.354889:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17067 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.354907:0:68465:0:(rw26.c:912:ll_write_end()) pos 69906432, len 4096, copied 4096 00000080:00200000:2.0:1769074919.354911:0:68465:0:(rw26.c:969:ll_write_end()) page@000000002ad192c2[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.354914:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000bcb9fa0b 17ffffc0001001 3:0 ffff9b03432fcbd0 17067 lru 00000080:00200000:2.0:1769074919.354920:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000046959b83 17067: 1< 0 - - > 2< 69906432 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.354923:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000002ad192c2 00000080:00200000:2.0:1769074919.354924:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 172. 00000080:00200000:2.0:1769074919.354926:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17068 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.354946:0:68465:0:(rw26.c:912:ll_write_end()) pos 69910528, len 4096, copied 4096 00000080:00200000:2.0:1769074919.354951:0:68465:0:(rw26.c:969:ll_write_end()) page@000000007605d116[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.354954:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000083c9e4e1 17ffffc0001001 3:0 ffff9b03432fc870 17068 lru 00000080:00200000:2.0:1769074919.354960:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000013b3db5e 17068: 1< 0 - - > 2< 69910528 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.354962:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000007605d116 00000080:00200000:2.0:1769074919.354963:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 173. 00000080:00200000:2.0:1769074919.354966:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17069 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.354979:0:68465:0:(rw26.c:912:ll_write_end()) pos 69914624, len 4096, copied 4096 00000080:00200000:2.0:1769074919.354984:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000069745a51[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.354987:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000aab99226 17ffffc0001001 3:0 ffff9b03432fc288 17069 lru 00000080:00200000:2.0:1769074919.354992:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000001a013fb0 17069: 1< 0 - - > 2< 69914624 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.354995:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000069745a51 00000080:00200000:2.0:1769074919.354996:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 174. 00000080:00200000:2.0:1769074919.354998:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17070 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355015:0:68465:0:(rw26.c:912:ll_write_end()) pos 69918720, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355020:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000935c95fe[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355023:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000ac5c75fd 17ffffc0001001 3:0 ffff9b03432fc0d8 17070 lru 00000080:00200000:2.0:1769074919.355028:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008b2b03bb 17070: 1< 0 - - > 2< 69918720 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355031:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000935c95fe 00000080:00200000:2.0:1769074919.355032:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 175. 00000080:00200000:2.0:1769074919.355034:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17071 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355051:0:68465:0:(rw26.c:912:ll_write_end()) pos 69922816, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355056:0:68465:0:(rw26.c:969:ll_write_end()) page@000000002f38d1bf[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355058:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000000c3a2f06 17ffffc0001001 3:0 ffff9b03432fc5e8 17071 lru 00000080:00200000:2.0:1769074919.355064:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000ec4865df 17071: 1< 0 - - > 2< 69922816 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355066:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000002f38d1bf 00000080:00200000:2.0:1769074919.355068:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 176. 00000080:00200000:2.0:1769074919.355070:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17072 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355085:0:68465:0:(rw26.c:912:ll_write_end()) pos 69926912, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355089:0:68465:0:(rw26.c:969:ll_write_end()) page@000000001190c9c4[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355092:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000bd9ccacc 17ffffc0001001 3:0 ffff9b03432fcca8 17072 lru 00000080:00200000:2.0:1769074919.355097:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000481f8caf 17072: 1< 0 - - > 2< 69926912 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355099:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000001190c9c4 00000080:00200000:2.0:1769074919.355100:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 177. 00000080:00200000:2.0:1769074919.355101:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17073 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355116:0:68465:0:(rw26.c:912:ll_write_end()) pos 69931008, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355120:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006d007bd5[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355122:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000004f8c5eb1 17ffffc0001001 3:0 ffff9b03432fc000 17073 lru 00000080:00200000:2.0:1769074919.355127:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000004dc24631 17073: 1< 0 - - > 2< 69931008 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355129:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006d007bd5 00000080:00200000:2.0:1769074919.355130:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 178. 00000080:00200000:2.0:1769074919.355132:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17074 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355146:0:68465:0:(rw26.c:912:ll_write_end()) pos 69935104, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355150:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000078d59db1[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355153:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000859b3827 17ffffc0001001 3:0 ffff9b03432fc948 17074 lru 00000080:00200000:2.0:1769074919.355157:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000c147fdfd 17074: 1< 0 - - > 2< 69935104 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355160:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000078d59db1 00000080:00200000:2.0:1769074919.355161:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 179. 00000080:00200000:2.0:1769074919.355162:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17075 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355189:0:68465:0:(rw26.c:912:ll_write_end()) pos 69939200, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355193:0:68465:0:(rw26.c:969:ll_write_end()) page@000000007f4b1c0e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355196:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000008966cb99 17ffffc0001021 2:0 ffff9b03432fc510 17075 lru 00000080:00200000:2.0:1769074919.355202:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000005a5b21e7 17075: 1< 0 - - > 2< 69939200 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355204:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000007f4b1c0e 00000080:00200000:2.0:1769074919.355205:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 180. 00000080:00200000:2.0:1769074919.355207:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17076 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355227:0:68465:0:(rw26.c:912:ll_write_end()) pos 69943296, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355258:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a471f69c[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355261:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d6ae5288 17ffffc0001001 3:0 ffff9b035db195e8 17076 lru 00000080:00200000:2.0:1769074919.355268:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000fbed353a 17076: 1< 0 - - > 2< 69943296 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355270:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a471f69c 00000080:00200000:2.0:1769074919.355272:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 181. 00000080:00200000:2.0:1769074919.355275:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17077 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355293:0:68465:0:(rw26.c:912:ll_write_end()) pos 69947392, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355299:0:68465:0:(rw26.c:969:ll_write_end()) page@000000002a906470[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355302:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000009338c27 17ffffc0001001 3:0 ffff9b035db19ca8 17077 lru 00000080:00200000:2.0:1769074919.355308:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000000d49184f 17077: 1< 0 - - > 2< 69947392 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355311:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000002a906470 00000080:00200000:2.0:1769074919.355312:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 182. 00000080:00200000:2.0:1769074919.355313:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17078 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355328:0:68465:0:(rw26.c:912:ll_write_end()) pos 69951488, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355333:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006cd2e261[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355335:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000066ebee0a 17ffffc0001001 3:0 ffff9b035db19000 17078 lru 00000080:00200000:2.0:1769074919.355341:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008b344455 17078: 1< 0 - - > 2< 69951488 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355344:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006cd2e261 00000080:00200000:2.0:1769074919.355345:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 183. 00000080:00200000:2.0:1769074919.355347:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17079 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355363:0:68465:0:(rw26.c:912:ll_write_end()) pos 69955584, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355367:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000f37217c2[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355370:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b756e636 17ffffc0001001 3:0 ffff9b035db19948 17079 lru 00000080:00200000:2.0:1769074919.355375:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000dd506341 17079: 1< 0 - - > 2< 69955584 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355377:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000f37217c2 00000080:00200000:2.0:1769074919.355378:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 184. 00000080:00200000:2.0:1769074919.355380:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17080 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355397:0:68465:0:(rw26.c:912:ll_write_end()) pos 69959680, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355400:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a6864a66[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355402:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000a907ea2e 17ffffc0001001 3:0 ffff9b035db19510 17080 lru 00000080:00200000:2.0:1769074919.355407:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000661350b2 17080: 1< 0 - - > 2< 69959680 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355410:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a6864a66 00000080:00200000:2.0:1769074919.355411:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 185. 00000080:00200000:2.0:1769074919.355413:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17081 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355427:0:68465:0:(rw26.c:912:ll_write_end()) pos 69963776, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355431:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006fdfcace[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355434:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000092cad5dc 17ffffc0001001 3:0 ffff9b035db19438 17081 lru 00000080:00200000:2.0:1769074919.355439:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f0788ea6 17081: 1< 0 - - > 2< 69963776 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355441:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006fdfcace 00000080:00200000:2.0:1769074919.355443:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 186. 00000080:00200000:2.0:1769074919.355444:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17082 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355462:0:68465:0:(rw26.c:912:ll_write_end()) pos 69967872, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355467:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000bbce7659[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355470:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000ef24d1cb 17ffffc0001001 3:0 ffff9b035db19e58 17082 lru 00000080:00200000:2.0:1769074919.355476:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000cf886709 17082: 1< 0 - - > 2< 69967872 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355478:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000bbce7659 00000080:00200000:2.0:1769074919.355479:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 187. 00000080:00200000:2.0:1769074919.355482:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17083 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.355496:0:68465:0:(rw26.c:912:ll_write_end()) pos 69971968, len 4096, copied 4096 00000080:00200000:2.0:1769074919.355500:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000044eef322[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.355503:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000008ba60cb5 17ffffc0001001 3:0 ffff9b035db19d80 17083 lru 00000080:00200000:2.0:1769074919.355509:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000048de8b13 17083: 1< 0 - - > 2< 69971968 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.355512:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000044eef322 00000080:00200000:2.0:1769074919.355513:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 188. 00000080:00200000:2.0:1769074919.360402:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17084 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.360509:0:68465:0:(rw26.c:912:ll_write_end()) pos 69976064, len 4096, copied 4096 00000080:00200000:2.0:1769074919.360775:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004df4a31b[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.360779:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b0c1f47f 17ffffc0001001 3:0 ffff9b035db19af8 17084 lru 00000080:00200000:2.0:1769074919.360788:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000b9065ad0 17084: 1< 0 - - > 2< 69976064 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.360790:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004df4a31b 00000080:00200000:2.0:1769074919.360792:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 189. 00000080:00200000:2.0:1769074919.360796:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17085 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.360816:0:68465:0:(rw26.c:912:ll_write_end()) pos 69980160, len 4096, copied 4096 00000080:00200000:2.0:1769074919.360820:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000dd5bf193[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.360822:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000462eb27c 17ffffc0001001 3:0 ffff9b035db19798 17085 lru 00000080:00200000:2.0:1769074919.360828:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000016db2957 17085: 1< 0 - - > 2< 69980160 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.360830:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000dd5bf193 00000080:00200000:2.0:1769074919.360831:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 190. 00000080:00200000:2.0:1769074919.360833:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17086 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.360849:0:68465:0:(rw26.c:912:ll_write_end()) pos 69984256, len 4096, copied 4096 00000080:00200000:2.0:1769074919.360853:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000026053d5d[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.360856:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000003e114f88 17ffffc0001001 3:0 ffff9b035db191b0 17086 lru 00000080:00200000:2.0:1769074919.360861:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f707b379 17086: 1< 0 - - > 2< 69984256 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.360863:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000026053d5d 00000080:00200000:2.0:1769074919.360864:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 191. 00000080:00200000:2.0:1769074919.360866:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17087 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.360888:0:68465:0:(rw26.c:912:ll_write_end()) pos 69988352, len 4096, copied 4096 00000080:00200000:2.0:1769074919.360893:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b6ebc679[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.360895:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000053908951 17ffffc0001001 3:0 ffff9b035db19a20 17087 lru 00000080:00200000:2.0:1769074919.360900:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e6089492 17087: 1< 0 - - > 2< 69988352 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.360903:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b6ebc679 00000080:00200000:2.0:1769074919.360904:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 192. 00000080:00200000:2.0:1769074919.360906:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17088 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.360930:0:68465:0:(rw26.c:912:ll_write_end()) pos 69992448, len 4096, copied 4096 00000080:00200000:2.0:1769074919.360934:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b9038db6[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.360936:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000024bf85c1 17ffffc0001001 3:0 ffff9b035db19360 17088 lru 00000080:00200000:2.0:1769074919.360941:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000001c4ed94e 17088: 1< 0 - - > 2< 69992448 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.360943:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b9038db6 00000080:00200000:2.0:1769074919.360944:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 193. 00000080:00200000:2.0:1769074919.360946:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17089 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.360964:0:68465:0:(rw26.c:912:ll_write_end()) pos 69996544, len 4096, copied 4096 00000080:00200000:2.0:1769074919.360968:0:68465:0:(rw26.c:969:ll_write_end()) page@000000005a10c5d8[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.360971:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000037631e84 17ffffc0001001 3:0 ffff9b035db196c0 17089 lru 00000080:00200000:2.0:1769074919.360975:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000a7858f7f 17089: 1< 0 - - > 2< 69996544 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.360978:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000005a10c5d8 00000080:00200000:2.0:1769074919.360979:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 194. 00000080:00200000:2.0:1769074919.360981:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17090 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361009:0:68465:0:(rw26.c:912:ll_write_end()) pos 70000640, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361013:0:68465:0:(rw26.c:969:ll_write_end()) page@000000001c3903c6[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361016:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000900fcb6c 17ffffc0001021 2:0 ffff9b035db19bd0 17090 lru 00000080:00200000:2.0:1769074919.361021:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000006c520f06 17090: 1< 0 - - > 2< 70000640 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361023:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000001c3903c6 00000080:00200000:2.0:1769074919.361024:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 195. 00000080:00200000:2.0:1769074919.361026:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17091 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361043:0:68465:0:(rw26.c:912:ll_write_end()) pos 70004736, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361047:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b294db0d[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361050:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000420cf36a 17ffffc0001001 3:0 ffff9b035db19870 17091 lru 00000080:00200000:2.0:1769074919.361054:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e37d851a 17091: 1< 0 - - > 2< 70004736 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361057:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b294db0d 00000080:00200000:2.0:1769074919.361058:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 196. 00000080:00200000:2.0:1769074919.361060:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17092 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361075:0:68465:0:(rw26.c:912:ll_write_end()) pos 70008832, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361079:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000079b988d2[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361082:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000004157fed0 17ffffc0001001 3:0 ffff9b035db19288 17092 lru 00000080:00200000:2.0:1769074919.361088:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000098bb9d67 17092: 1< 0 - - > 2< 70008832 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361090:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000079b988d2 00000080:00200000:2.0:1769074919.361091:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 197. 00000080:00200000:2.0:1769074919.361093:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17093 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361109:0:68465:0:(rw26.c:912:ll_write_end()) pos 70012928, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361112:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000deccfeaa[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361115:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d49d0fe2 17ffffc0001001 3:0 ffff9b035db190d8 17093 lru 00000080:00200000:2.0:1769074919.361120:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000008c0c1f38 17093: 1< 0 - - > 2< 70012928 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361122:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000deccfeaa 00000080:00200000:2.0:1769074919.361123:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 198. 00000080:00200000:2.0:1769074919.361125:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17094 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361142:0:68465:0:(rw26.c:912:ll_write_end()) pos 70017024, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361146:0:68465:0:(rw26.c:969:ll_write_end()) page@000000001a7812df[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361149:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b32fc18b 17ffffc0001001 3:0 ffff9b0372e64360 17094 lru 00000080:00200000:2.0:1769074919.361154:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000097a7eddb 17094: 1< 0 - - > 2< 70017024 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361156:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000001a7812df 00000080:00200000:2.0:1769074919.361157:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 199. 00000080:00200000:2.0:1769074919.361159:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17095 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361173:0:68465:0:(rw26.c:912:ll_write_end()) pos 70021120, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361178:0:68465:0:(rw26.c:969:ll_write_end()) page@000000006bbd7fe9[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361180:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000009ca9fdb5 17ffffc0001001 3:0 ffff9b0372e64a20 17095 lru 00000080:00200000:2.0:1769074919.361186:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000db2226d6 17095: 1< 0 - - > 2< 70021120 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361188:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000006bbd7fe9 00000080:00200000:2.0:1769074919.361189:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 200. 00000080:00200000:2.0:1769074919.361191:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17096 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361212:0:68465:0:(rw26.c:912:ll_write_end()) pos 70025216, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361216:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000001c58dc8[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361219:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000ac42a619 17ffffc0001001 3:0 ffff9b0372e641b0 17096 lru 00000080:00200000:2.0:1769074919.361224:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d7fcb8fb 17096: 1< 0 - - > 2< 70025216 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361226:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000001c58dc8 00000080:00200000:2.0:1769074919.361227:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 201. 00000080:00200000:2.0:1769074919.361229:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17097 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361279:0:68465:0:(rw26.c:912:ll_write_end()) pos 70029312, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361283:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000230e0de3[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361286:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000005a68fe8b 17ffffc0001001 3:0 ffff9b0372e64798 17097 lru 00000080:00200000:2.0:1769074919.361291:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000083de77a3 17097: 1< 0 - - > 2< 70029312 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361293:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000230e0de3 00000080:00200000:2.0:1769074919.361294:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 202. 00000080:00200000:2.0:1769074919.361296:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17098 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361312:0:68465:0:(rw26.c:912:ll_write_end()) pos 70033408, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361317:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000666986df[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361320:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000031c64df0 17ffffc0001001 3:0 ffff9b0372e64af8 17098 lru 00000080:00200000:2.0:1769074919.361325:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000024b5ce98 17098: 1< 0 - - > 2< 70033408 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361328:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000666986df 00000080:00200000:2.0:1769074919.361329:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 203. 00000080:00200000:2.0:1769074919.361331:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17099 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361347:0:68465:0:(rw26.c:912:ll_write_end()) pos 70037504, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361352:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000c47b9790[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361355:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000ad4fa888 17ffffc0001001 3:0 ffff9b0372e64d80 17099 lru 00000080:00200000:2.0:1769074919.361360:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e2b56928 17099: 1< 0 - - > 2< 70037504 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361362:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000c47b9790 00000080:00200000:2.0:1769074919.361364:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 204. 00000080:00200000:2.0:1769074919.361366:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17100 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361383:0:68465:0:(rw26.c:912:ll_write_end()) pos 70041600, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361387:0:68465:0:(rw26.c:969:ll_write_end()) page@000000001f25230e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361390:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b7d783ad 17ffffc0001001 3:0 ffff9b0372e64e58 17100 lru 00000080:00200000:2.0:1769074919.361395:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000b8e8001f 17100: 1< 0 - - > 2< 70041600 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361398:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000001f25230e 00000080:00200000:2.0:1769074919.361399:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 205. 00000080:00200000:2.0:1769074919.361401:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17101 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361417:0:68465:0:(rw26.c:912:ll_write_end()) pos 70045696, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361422:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000f05fb550[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361425:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000c8c74b77 17ffffc0001001 3:0 ffff9b0372e64438 17101 lru 00000080:00200000:2.0:1769074919.361429:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000073e80e46 17101: 1< 0 - - > 2< 70045696 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361431:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000f05fb550 00000080:00200000:2.0:1769074919.361432:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 206. 00000080:00200000:2.0:1769074919.361435:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17102 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361453:0:68465:0:(rw26.c:912:ll_write_end()) pos 70049792, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361457:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000f6d303eb[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361460:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000044e7b3a9 17ffffc0001001 3:0 ffff9b0372e64510 17102 lru 00000080:00200000:2.0:1769074919.361466:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000e56eb26a 17102: 1< 0 - - > 2< 70049792 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361468:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000f6d303eb 00000080:00200000:2.0:1769074919.361469:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 207. 00000080:00200000:2.0:1769074919.361471:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17103 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.361491:0:68465:0:(rw26.c:912:ll_write_end()) pos 70053888, len 4096, copied 4096 00000080:00200000:2.0:1769074919.361495:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000091d8ab5c[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.361497:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000023c96169 17ffffc0001001 3:0 ffff9b0372e64948 17103 lru 00000080:00200000:2.0:1769074919.361502:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000001205b353 17103: 1< 0 - - > 2< 70053888 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.361504:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000091d8ab5c 00000080:00200000:2.0:1769074919.361505:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 208. 00000080:00200000:2.0:1769074919.361507:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17104 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.363823:0:68465:0:(rw26.c:912:ll_write_end()) pos 70057984, len 4096, copied 4096 00000080:00200000:2.0:1769074919.363842:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000055f91e93[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.363845:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d6779b28 17ffffc0001001 3:0 ffff9b0372e646c0 17104 lru 00000080:00200000:2.0:1769074919.363853:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d4f9c52a 17104: 1< 0 - - > 2< 70057984 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.363856:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000055f91e93 00000080:00200000:2.0:1769074919.363857:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 209. 00000080:00200000:2.0:1769074919.363861:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17105 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.363913:0:68465:0:(rw26.c:912:ll_write_end()) pos 70062080, len 4096, copied 4096 00000080:00200000:2.0:1769074919.363917:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a8de4143[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.363920:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000258eb63b 17ffffc0001021 2:0 ffff9b0372e64bd0 17105 lru 00000080:00200000:2.0:1769074919.363926:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000ec7a0e22 17105: 1< 0 - - > 2< 70062080 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.363928:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a8de4143 00000080:00200000:2.0:1769074919.363930:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 210. 00000080:00200000:2.0:1769074919.363932:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17106 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.363949:0:68465:0:(rw26.c:912:ll_write_end()) pos 70066176, len 4096, copied 4096 00000080:00200000:2.0:1769074919.363953:0:68465:0:(rw26.c:969:ll_write_end()) page@000000005b59309a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.363956:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000004fdf25f3 17ffffc0001001 3:0 ffff9b0372e64000 17106 lru 00000080:00200000:2.0:1769074919.363961:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000162c6cf9 17106: 1< 0 - - > 2< 70066176 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.363963:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000005b59309a 00000080:00200000:2.0:1769074919.363964:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 211. 00000080:00200000:2.0:1769074919.363966:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17107 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.363983:0:68465:0:(rw26.c:912:ll_write_end()) pos 70070272, len 4096, copied 4096 00000080:00200000:2.0:1769074919.363987:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000994ab114[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.363990:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000007c76745 17ffffc0001001 3:0 ffff9b0372e64ca8 17107 lru 00000080:00200000:2.0:1769074919.363995:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f33eb746 17107: 1< 0 - - > 2< 70070272 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.363998:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000994ab114 00000080:00200000:2.0:1769074919.363999:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 212. 00000080:00200000:2.0:1769074919.364001:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17108 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364019:0:68465:0:(rw26.c:912:ll_write_end()) pos 70074368, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364024:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000ea9f851b[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364026:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000002c76eaf6 17ffffc0001001 3:0 ffff9b0372e645e8 17108 lru 00000080:00200000:2.0:1769074919.364032:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000aff6a64f 17108: 1< 0 - - > 2< 70074368 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364034:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000ea9f851b 00000080:00200000:2.0:1769074919.364036:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 213. 00000080:00200000:2.0:1769074919.364038:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17109 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364054:0:68465:0:(rw26.c:912:ll_write_end()) pos 70078464, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364059:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000b8cec455[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364062:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000427d3e5d 17ffffc0001001 3:0 ffff9b0372e640d8 17109 lru 00000080:00200000:2.0:1769074919.364067:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000014e69cda 17109: 1< 0 - - > 2< 70078464 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364069:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000b8cec455 00000080:00200000:2.0:1769074919.364070:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 214. 00000080:00200000:2.0:1769074919.364072:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17110 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364092:0:68465:0:(rw26.c:912:ll_write_end()) pos 70082560, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364096:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000061830844[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364099:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000cc59f3cb 17ffffc0001001 3:0 ffff9b0372e64288 17110 lru 00000080:00200000:2.0:1769074919.364105:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f94142d8 17110: 1< 0 - - > 2< 70082560 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364107:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000061830844 00000080:00200000:2.0:1769074919.364108:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 215. 00000080:00200000:2.0:1769074919.364110:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17111 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364126:0:68465:0:(rw26.c:912:ll_write_end()) pos 70086656, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364130:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000084d92d4a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364133:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000008ef84986 17ffffc0001001 3:0 ffff9b0372e64870 17111 lru 00000080:00200000:2.0:1769074919.364138:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000aad0e269 17111: 1< 0 - - > 2< 70086656 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364140:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000084d92d4a 00000080:00200000:2.0:1769074919.364141:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 216. 00000080:00200000:2.0:1769074919.364143:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17112 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364161:0:68465:0:(rw26.c:912:ll_write_end()) pos 70090752, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364166:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000bf11e7af[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364168:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000008ce4a50a 17ffffc0001001 3:0 ffff9b035086d1b0 17112 lru 00000080:00200000:2.0:1769074919.364174:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000206c4298 17112: 1< 0 - - > 2< 70090752 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364176:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000bf11e7af 00000080:00200000:2.0:1769074919.364177:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 217. 00000080:00200000:2.0:1769074919.364180:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17113 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364196:0:68465:0:(rw26.c:912:ll_write_end()) pos 70094848, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364201:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000e2777c3a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364203:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000005d4f5e3f 17ffffc0001001 3:0 ffff9b035086d798 17113 lru 00000080:00200000:2.0:1769074919.364208:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000035da307b 17113: 1< 0 - - > 2< 70094848 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364210:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000e2777c3a 00000080:00200000:2.0:1769074919.364211:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 218. 00000080:00200000:2.0:1769074919.364213:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17114 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364251:0:68465:0:(rw26.c:912:ll_write_end()) pos 70098944, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364255:0:68465:0:(rw26.c:969:ll_write_end()) page@000000008848198e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364258:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000087e92bc 17ffffc0001001 3:0 ffff9b035086daf8 17114 lru 00000080:00200000:2.0:1769074919.364263:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000002961860d 17114: 1< 0 - - > 2< 70098944 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364265:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000008848198e 00000080:00200000:2.0:1769074919.364266:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 219. 00000080:00200000:2.0:1769074919.364269:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17115 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364287:0:68465:0:(rw26.c:912:ll_write_end()) pos 70103040, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364291:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000dc91bbae[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364294:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b19bd1c9 17ffffc0001001 3:0 ffff9b035086dd80 17115 lru 00000080:00200000:2.0:1769074919.364300:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000f843764c 17115: 1< 0 - - > 2< 70103040 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364302:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000dc91bbae 00000080:00200000:2.0:1769074919.364303:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 220. 00000080:00200000:2.0:1769074919.364305:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17116 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364320:0:68465:0:(rw26.c:912:ll_write_end()) pos 70107136, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364325:0:68465:0:(rw26.c:969:ll_write_end()) page@000000008dedc018[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364328:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000003d89f074 17ffffc0001001 3:0 ffff9b035086de58 17116 lru 00000080:00200000:2.0:1769074919.364332:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000071b0cceb 17116: 1< 0 - - > 2< 70107136 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364335:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000008dedc018 00000080:00200000:2.0:1769074919.364336:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 221. 00000080:00200000:2.0:1769074919.364338:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17117 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364354:0:68465:0:(rw26.c:912:ll_write_end()) pos 70111232, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364358:0:68465:0:(rw26.c:969:ll_write_end()) page@000000004206fe7f[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364361:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000d7420757 17ffffc0001001 3:0 ffff9b035086d438 17117 lru 00000080:00200000:2.0:1769074919.364366:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000a0cd07ce 17117: 1< 0 - - > 2< 70111232 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364368:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000004206fe7f 00000080:00200000:2.0:1769074919.364370:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 222. 00000080:00200000:2.0:1769074919.364372:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17118 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364387:0:68465:0:(rw26.c:912:ll_write_end()) pos 70115328, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364391:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000d336c633[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364394:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000065d19fdb 17ffffc0001001 3:0 ffff9b035086d510 17118 lru 00000080:00200000:2.0:1769074919.364399:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000ca46414c 17118: 1< 0 - - > 2< 70115328 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364402:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000d336c633 00000080:00200000:2.0:1769074919.364403:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 223. 00000080:00200000:2.0:1769074919.364405:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17119 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364422:0:68465:0:(rw26.c:912:ll_write_end()) pos 70119424, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364425:0:68465:0:(rw26.c:969:ll_write_end()) page@000000003183fd6e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364428:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @0000000024536373 17ffffc0001001 3:0 ffff9b035086d948 17119 lru 00000080:00200000:2.0:1769074919.364433:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000007ba4ff59 17119: 1< 0 - - > 2< 70119424 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364435:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000003183fd6e 00000080:00200000:2.0:1769074919.364436:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 224. 00000080:00200000:2.0:1769074919.364438:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17120 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364459:0:68465:0:(rw26.c:912:ll_write_end()) pos 70123520, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364464:0:68465:0:(rw26.c:969:ll_write_end()) page@000000001c1f785a[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364466:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000f6e601e7 17ffffc0001021 2:0 ffff9b034120fe58 17120 lru 00000080:00200000:2.0:1769074919.364472:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000000b19cac3 17120: 1< 0 - - > 2< 70123520 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364474:0:68465:0:(rw26.c:969:ll_write_end()) end page@000000001c1f785a 00000080:00200000:2.0:1769074919.364475:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 225. 00000080:00200000:2.0:1769074919.364477:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17121 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.364495:0:68465:0:(rw26.c:912:ll_write_end()) pos 70127616, len 4096, copied 4096 00000080:00200000:2.0:1769074919.364499:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000062cf2fc5[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.364502:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000b418a080 17ffffc0001001 3:0 ffff9b034120fd80 17121 lru 00000080:00200000:2.0:1769074919.364507:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000d669c721 17121: 1< 0 - - > 2< 70127616 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.364509:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000062cf2fc5 00000080:00200000:2.0:1769074919.364510:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 226. 00000080:00200000:2.0:1769074919.364512:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17122 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.367452:0:68465:0:(rw26.c:912:ll_write_end()) pos 70131712, len 4096, copied 4096 00000080:00200000:2.0:1769074919.367477:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000a2fe7bd9[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.367482:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000afdd5626 17ffffc0001001 3:0 ffff9b034120faf8 17122 lru 00000080:00200000:2.0:1769074919.367490:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@000000004fd1a4ae 17122: 1< 0 - - > 2< 70131712 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.367493:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000a2fe7bd9 00000080:00200000:2.0:1769074919.367495:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 227. 00000080:00200000:2.0:1769074919.367501:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17123 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.368144:0:68465:0:(rw26.c:912:ll_write_end()) pos 70135808, len 4096, copied 4096 00000080:00200000:2.0:1769074919.368152:0:68465:0:(rw26.c:969:ll_write_end()) page@0000000006c135d5[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.368155:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000006f924f5c 17ffffc0001001 3:0 ffff9b034120f798 17123 lru 00000080:00200000:2.0:1769074919.368161:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@00000000864e2045 17123: 1< 0 - - > 2< 70135808 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.368164:0:68465:0:(rw26.c:969:ll_write_end()) end page@0000000006c135d5 00000080:00200000:2.0:1769074919.368165:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 228. 00000080:00200000:2.0:1769074919.368167:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17124 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.368193:0:68465:0:(rw26.c:912:ll_write_end()) pos 70139904, len 4096, copied 4096 00000080:00200000:2.0:1769074919.368197:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000f2ef7914[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.368200:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @000000006d773a94 17ffffc0001001 3:0 ffff9b034120f1b0 17124 lru 00000080:00200000:2.0:1769074919.368205:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000087cba939 17124: 1< 0 - - > 2< 70139904 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.368207:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000f2ef7914 00000080:00200000:2.0:1769074919.368208:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 229. 00000080:00200000:2.0:1769074919.368210:0:68465:0:(rw26.c:729:ll_write_begin()) Writing 17125 of 0 to 4096 bytes 00000080:00200000:2.0:1769074919.368225:0:68465:0:(rw26.c:912:ll_write_end()) pos 70144000, len 4096, copied 4096 00000080:00200000:2.0:1769074919.368229:0:68465:0:(rw26.c:969:ll_write_end()) page@00000000f0f47a2e[3 000000001d268f5f 2 1 00000000c8bb8e26] 00000080:00200000:2.0:1769074919.368231:0:68465:0:(rw26.c:969:ll_write_end()) vmpage @00000000643862dd 17ffffc0001001 3:0 ffff9b034120fa20 17125 lru 00000080:00200000:2.0:1769074919.368496:0:68465:0:(rw26.c:969:ll_write_end()) osc-page@0000000098d47a0f 17125: 1< 0 - - > 2< 70144000 0 0 0x0 0x0 | 00000000e7925d05 00000000dd897576 > 3< 0 0 > 4< 0 0 8 51576832 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1769074919.368499:0:68465:0:(rw26.c:969:ll_write_end()) end page@00000000f0f47a2e 00000080:00200000:2.0:1769074919.368501:0:68465:0:(rw26.c:969:ll_write_end()) queued page: 230. 00000008:00010000:0.0:1769074919.378404:0:68465:0:(osc_cache.c:814:osc_extent_find()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.378455:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.378464:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.378474:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.378482:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.378489:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.378497:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.378510:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379095:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379105:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379114:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379122:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379131:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379139:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379147:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.379177:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.379188:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379198:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379206:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379390:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379405:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379414:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379423:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379433:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379441:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379449:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379464:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379475:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379487:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379494:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.379502:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.380090:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.380106:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380119:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380127:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380135:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380144:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380152:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380160:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380168:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380176:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380183:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380192:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380200:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380207:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380427:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380436:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.380451:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.380456:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380464:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380471:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380477:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380484:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380490:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380496:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380503:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.380510:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382356:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382384:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382394:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382403:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382413:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382420:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.382443:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.382455:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382465:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382474:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382483:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382492:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382499:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.382508:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.386932:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.386946:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.386954:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.386968:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.386976:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.386994:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387002:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387011:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.387039:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.387049:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387057:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387065:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387073:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387083:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387091:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387098:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387106:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387113:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387120:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387127:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387135:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387142:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387155:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387162:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.387176:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.387181:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387189:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387196:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387205:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387213:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387221:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387228:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387253:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387261:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387269:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387276:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387283:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387291:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387298:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387305:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.387318:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.387322:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387331:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387339:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387347:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387355:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387362:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387370:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387378:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387385:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387392:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387405:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387412:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387419:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387427:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387434:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.387447:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.387451:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387458:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387466:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387473:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387481:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387488:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387496:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387503:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387510:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387518:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387525:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387532:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387539:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387551:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387558:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.387570:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.387574:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387582:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387589:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387597:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387604:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387612:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387619:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387628:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387645:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387655:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387662:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387670:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387677:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387684:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387692:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.387704:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.387709:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387718:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387726:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387733:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387740:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387747:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387754:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387762:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387769:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387776:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387789:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387796:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387804:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387812:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387820:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.387832:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.387837:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387845:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387852:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387859:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387867:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387874:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387881:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387888:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387896:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387903:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387911:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387918:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387924:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387938:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387946:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.387958:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.387963:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387971:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387978:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387985:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.387993:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388001:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388008:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388017:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388025:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388033:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388040:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388046:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388053:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388060:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388067:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.388079:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.388084:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388093:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388100:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388108:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388116:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388123:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388131:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388138:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388146:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388153:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388165:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388172:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388180:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388187:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388194:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.388206:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.388211:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388218:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388226:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388259:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388269:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388278:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388286:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388294:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388302:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388309:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388317:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388324:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388331:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388344:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388353:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.388366:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.388370:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388378:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388385:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388394:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388402:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388410:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388417:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388426:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388434:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388441:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388448:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388455:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388462:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388470:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388477:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.388489:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.388494:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388501:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388508:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388516:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388523:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388530:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388537:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388544:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388553:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388561:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388573:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388582:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388589:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388596:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:0.0:1769074919.388604:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.388616:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 15, dirtied 15 00000008:00010000:0.0:1769074919.388621:0:68465:0:(osc_cache.c:2423:osc_queue_async_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769074919.388629:0:68465:0:(vvp_io.c:1106:vvp_set_batch_dirty()) mapping ffff9b0355ce2378, count 1, dirtied 1 00000080:00200000:0.0:1769074919.388634:0:68465:0:(vvp_io.c:1216:vvp_io_write_commit()) Committed 256 pages 1048576 bytes, tot: 1048576 00000080:00200000:0.0:1769074919.391408:0:68465:0:(vvp_io.c:1390:vvp_io_write_start()) f204d.sanity-flr: write bytes 0, result: 1048576 00000020:00010000:0.0:1769074919.391451:0:68465:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b03592837b8 (ffff9b035121f758/0) at cl_lock_release():245 00000020:00010000:0.0:1769074919.391457:0:68465:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b03592837b8 (ffff9b035121f758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074919.391463:0:68465:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359274330 (ffff9b0342ca8a28/1) at cl_lock_cancel():142 00010000:00010000:0.0:1769074919.391475:0:68465:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074919.391491:0:68465:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000020:00010000:0.0:1769074919.391511:0:68465:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b03592837b8 (ffff9b035121f758/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074919.391592:0:68465:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359274330 (ffff9b035121f758/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074919.391611:0:68465:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00000080:00200000:0.0:1769074919.391619:0:68465:0:(file.c:2109:ll_file_io_generic()) Range unlock [16896, 17151] 00000080:00200000:0.0:1769074919.391629:0:68465:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 6 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074919.391632:0:68465:0:(file.c:2142:ll_file_io_generic()) f204d.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:1.0:1769074919.393203:0:68465:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 0 00000080:00200000:1.0:1769074919.393376:0:68465:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:1.0:1769074919.393441:0:68465:0:(file.c:424:ll_file_release()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8), flags 100002 00000002:00100000:1.0:1769074919.394126:0:68465:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0342ceb480 x1855007617058048/t4294968378(4294968378) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 984/1056 e 0 to 0 dl 1769074974 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1769074919.394145:0:68465:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1769074919.394151:0:68465:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ce9180 x1855007617058816/t0(0) o35->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2192 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1769074919.394189:0:68465:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68465:1855007617058816:192.168.204.160@tcp:35:dd.0 00000100:00100000:1.0:1769074919.394472:0:68465:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a68f00 going to sleep for 11 seconds 00000100:00100000:2.0:1769074919.415667:0:68465:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job dd:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68465:1855007617058816:192.168.204.160@tcp:35:dd.0 00000002:00100000:2.0:1769074919.415727:0:68465:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0342ceb480 x1855007617058048/t4294968378(4294968378) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 984/1056 e 0 to 0 dl 1769074974 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1769074919.891914:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074920.915933:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074921.043730:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617059072:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074921.043852:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617059328:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074921.043889:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:0.0:1769074921.043895:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617058944:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074921.043919:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617059200:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074921.043936:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617059456:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074921.055974:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617058944:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074921.056000:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617059200:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074921.056013:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617059456:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074921.056025:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617059072:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074921.056035:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617059328:192.168.204.160@tcp:400:kworker.0 00010000:00010000:2.0:1769074921.939502:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074922.964632:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074927.059532:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074928.083553:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074929.107572:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074930.131521:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074931.283599:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617060352:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074931.283712:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617060608:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074931.283748:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1769074931.283754:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617060224:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074931.283782:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617060480:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074931.283821:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617060736:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074931.291212:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617060224:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074931.291288:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617060480:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074931.291301:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617060736:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074931.291312:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617060352:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074931.291322:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617060608:192.168.204.160@tcp:400:kworker.0 00000080:00200000:2.0:1769074932.389822:0:68488:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074932.389854:0:68488:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074932.389871:0:68488:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.389886:0:68488:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.389894:0:68488:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074932.389903:0:68488:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074932.389907:0:68488:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074932.389914:0:68488:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364241fbf8 00010000:00010000:2.0:1769074932.389918:0:68488:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.389926:0:68488:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.389947:0:68488:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074932.389956:0:68488:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:2.0:1769074932.389962:0:68488:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:2.0:1769074932.389974:0:68488:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b034532ed80, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074932.389981:0:68488:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open 00000002:00010000:2.0:1769074932.389998:0:68488:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074932.390053:0:68488:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.390062:0:68488:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68488 initiator: MDT0 00000020:00100000:2.0:1769074932.390074:0:68488:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074932.390087:0:68488:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7480 x1855007617060992/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074932.390097:0:68488:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68488 initiator: MDT0 00000100:00100000:2.0:1769074932.390119:0:68488:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68488:1855007617060992:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074932.391315:0:68488:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e480 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.399658:0:68488:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68488:1855007617060992:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.399682:0:68488:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.399696:0:68488:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.399707:0:68488:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.399714:0:68488:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.399721:0:68488:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.399746:0:68488:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860400 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074932.399751:0:68488:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.399758:0:68488:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.399764:0:68488:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 5/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.399771:0:68488:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074932.399777:0:68488:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.399783:0:68488:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9d6 lrc: 0/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68488 initiator: MDT0 00000002:00100000:2.0:1769074932.399810:0:68488:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b03482c7480 x1855007617060992/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/608 e 0 to 0 dl 1769074987 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074932.399825:0:68488:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x240000405:0xa:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074932.399955:0:68488:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074932.399976:0:68488:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074932.399990:0:68488:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.400003:0:68488:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.400010:0:68488:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074932.400016:0:68488:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074932.400020:0:68488:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074932.400025:0:68488:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364241fbf8 00010000:00010000:2.0:1769074932.400029:0:68488:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.400038:0:68488:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.400052:0:68488:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074932.400059:0:68488:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:2.0:1769074932.400066:0:68488:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 44 00000080:00200008:2.0:1769074932.400074:0:68488:0:(file.c:1001:ll_file_open()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000002:00010000:2.0:1769074932.400088:0:68488:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20c:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:2.0:1769074932.400132:0:68488:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.400139:0:68488:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68488 initiator: MDT0 00000020:00100000:2.0:1769074932.400149:0:68488:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074932.400156:0:68488:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7480 x1855007617061248/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074932.400170:0:68488:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68488 initiator: MDT0 00000100:00100000:2.0:1769074932.400189:0:68488:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68488:1855007617061248:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074932.400407:0:68488:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e480 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.406765:0:68488:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68488:1855007617061248:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.406787:0:68488:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.406801:0:68488:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.406808:0:68488:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.406815:0:68488:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.406822:0:68488:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.406829:0:68488:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860200 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074932.406832:0:68488:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.406840:0:68488:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.406846:0:68488:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.406853:0:68488:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074932.406859:0:68488:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68488 initiator: MDT0 00010000:00010000:2.0:1769074932.406865:0:68488:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9dd lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 68488 initiator: MDT0 00000002:00100000:2.0:1769074932.406883:0:68488:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9b03482c7480 x1855007617061248/t8589944142(8589944142) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074987 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074932.406897:0:68488:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b03482c7480 x1855007617061248/t8589944142(8589944142) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074987 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074932.406913:0:68488:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074932.406946:0:68488:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20c:0x0], updating i_size 4096 00000002:00100000:2.0:1769074932.406965:0:68488:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b03482c7480 x1855007617061248/t8589944142(8589944142) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074987 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1769074932.407055:0:68488:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988) cmd=c0086915 arg=7c23d0 00000100:00100000:2.0:1769074932.407114:0:68488:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68488:1855007617061376:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074932.407292:0:68488:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e480 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.425515:0:68488:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68488:1855007617061376:192.168.204.160@tcp:34:lfs.0 00000080:00200008:0.0:1769074932.432001:0:68488:0:(file.c:424:ll_file_release()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000002:00100000:0.0:1769074932.432087:0:68488:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b03482c7480 x1855007617061248/t8589944142(8589944142) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074987 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074932.432100:0:68488:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074932.432106:0:68488:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3a680 x1855007617061632/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074932.432130:0:68488:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68488:1855007617061632:192.168.204.160@tcp:35:lfs.0 00000100:00100000:0.0:1769074932.432222:0:68488:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0345854780 going to sleep for 11 seconds 00000100:00100000:0.0:1769074932.441363:0:68488:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68488:1855007617061632:192.168.204.160@tcp:35:lfs.0 00000002:00100000:0.0:1769074932.441401:0:68488:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b03482c7480 x1855007617061248/t8589944142(8589944142) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074987 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1769074932.481195:0:68490:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074932.481858:0:68490:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074932.481883:0:68490:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074932.481900:0:68490:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074932.481916:0:68490:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074932.481923:0:68490:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074932.481931:0:68490:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074932.481935:0:68490:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074932.481942:0:68490:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642347cb8 00010000:00010000:1.0:1769074932.481947:0:68490:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074932.481955:0:68490:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074932.482101:0:68490:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074932.482112:0:68490:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074932.482119:0:68490:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074932.482127:0:68490:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074932.482132:0:68490:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074932.482136:0:68490:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074932.482138:0:68490:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074932.482142:0:68490:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642347bb0 00010000:00010000:1.0:1769074932.482145:0:68490:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074932.482150:0:68490:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074932.482158:0:68490:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074932.482166:0:68490:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074932.482184:0:68490:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffd74ecf83c 00000080:00200008:1.0:1769074932.482196:0:68490:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:2.0:1769074932.484911:0:68493:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074932.484939:0:68493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074932.484957:0:68493:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.484972:0:68493:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.484980:0:68493:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074932.484989:0:68493:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074932.484993:0:68493:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074932.485001:0:68493:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642427bf8 00010000:00010000:2.0:1769074932.485005:0:68493:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.485013:0:68493:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.485028:0:68493:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074932.485035:0:68493:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:2.0:1769074932.485042:0:68493:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:2.0:1769074932.485054:0:68493:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b034532ed80, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074932.485060:0:68493:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open 00000002:00010000:2.0:1769074932.485077:0:68493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074932.485132:0:68493:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9e4 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68493 initiator: MDT0 00010000:00010000:2.0:1769074932.485139:0:68493:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9e4 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68493 initiator: MDT0 00000020:00100000:2.0:1769074932.485151:0:68493:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074932.485165:0:68493:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c5880 x1855007617061888/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074932.485174:0:68493:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879c9e4 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68493 initiator: MDT0 00000100:00100000:2.0:1769074932.485195:0:68493:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68493:1855007617061888:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074932.485410:0:68493:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e480 going to sleep for 50 seconds 00000080:00200000:0.0:1769074932.517673:0:68493:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074932.517707:0:68493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074932.517723:0:68493:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074932.517745:0:68493:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074932.517921:0:68493:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074932.517929:0:68493:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074932.517933:0:68493:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:0.0:1769074932.517940:0:68493:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642427bf8 00010000:00010000:0.0:1769074932.517944:0:68493:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074932.517952:0:68493:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074932.517966:0:68493:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074932.517974:0:68493:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:0.0:1769074932.517980:0:68493:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 44 00000080:00200008:0.0:1769074932.517989:0:68493:0:(file.c:1001:ll_file_open()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000002:00010000:0.0:1769074932.518006:0:68493:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20c:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240304001 00010000:00010000:0.0:1769074932.518060:0:68493:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68493 initiator: MDT0 00010000:00010000:0.0:1769074932.518067:0:68493:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68493 initiator: MDT0 00000020:00100000:0.0:1769074932.518079:0:68493:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074932.518091:0:68493:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3a680 x1855007617062144/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074932.518101:0:68493:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68493 initiator: MDT0 00000100:00100000:0.0:1769074932.518124:0:68493:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68493:1855007617062144:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074932.519735:0:68493:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0345854780 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.525458:0:68493:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68493:1855007617062144:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.525493:0:68493:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 4/1,0 mode: --/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.525505:0:68493:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 4/1,0 mode: --/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.525516:0:68493:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.525526:0:68493:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 4/1,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074932.525540:0:68493:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=200002b, status=0 req@ffff9b0365a3a680 x1855007617062144/t8589944144(8589944144) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074987 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074932.525552:0:68493:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0365a3a680 x1855007617062144/t8589944144(8589944144) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769074987 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074932.525567:0:68493:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.525576:0:68493:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 16 res 8589935621/524 (0 0) 00000080:00200000:2.0:1769074932.525582:0:68493:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074932.525614:0:68493:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20c:0x0], updating i_size 4096 00000080:00010000:2.0:1769074932.525622:0:68493:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20c:0x0](ffff9b0355d41988) for lock 0xb7d6bf619879c9eb 00000080:00010000:2.0:1769074932.525628:0:68493:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c9eb from it ffffb43642427bd0 00010000:00010000:2.0:1769074932.525632:0:68493:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 3/1,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.525641:0:68493:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074932.525664:0:68493:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0365a3a680 x1855007617062144/t8589944144(8589944144) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074987 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1769074932.525792:0:68493:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988) cmd=c1406916 arg=23ffa10 00000100:00100000:2.0:1769074932.525853:0:68493:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68493:1855007617062272:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074932.526055:0:68493:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ec00 going to sleep for 50 seconds 00000080:00200000:2.0:1769074932.574119:0:68496:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074932.574149:0:68496:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074932.574168:0:68496:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.574184:0:68496:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.574193:0:68496:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074932.574200:0:68496:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074932.574204:0:68496:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074932.574211:0:68496:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364241fbf8 00010000:00010000:2.0:1769074932.574215:0:68496:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.574223:0:68496:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.574575:0:68496:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074932.574585:0:68496:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:2.0:1769074932.574591:0:68496:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:2.0:1769074932.574604:0:68496:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b0348c18480, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074932.574610:0:68496:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open 00000002:00010000:2.0:1769074932.574627:0:68496:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074932.574684:0:68496:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68496 initiator: MDT0 00010000:00010000:2.0:1769074932.574692:0:68496:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68496 initiator: MDT0 00000020:00100000:2.0:1769074932.574703:0:68496:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074932.574715:0:68496:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c4380 x1855007617062528/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074932.574724:0:68496:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68496 initiator: MDT0 00000100:00100000:2.0:1769074932.574745:0:68496:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68496:1855007617062528:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074932.575403:0:68496:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e6c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.584719:0:68496:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68496:1855007617062528:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.584753:0:68496:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68496 initiator: MDT0 00010000:00010000:2.0:1769074932.584767:0:68496:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68496 initiator: MDT0 00010000:00010000:2.0:1769074932.584773:0:68496:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68496 initiator: MDT0 00010000:00010000:2.0:1769074932.584780:0:68496:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68496 initiator: MDT0 00010000:00010000:2.0:1769074932.584792:0:68496:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68496 initiator: MDT0 00010000:00010000:2.0:1769074932.584799:0:68496:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860400 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074932.584803:0:68496:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68496 initiator: MDT0 00010000:00010000:2.0:1769074932.584810:0:68496:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68496 initiator: MDT0 00010000:00010000:2.0:1769074932.584816:0:68496:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 5/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 68496 initiator: MDT0 00010000:00010000:2.0:1769074932.584823:0:68496:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074932.584828:0:68496:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68496 initiator: MDT0 00010000:00010000:2.0:1769074932.584834:0:68496:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879c9f2 lrc: 0/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68496 initiator: MDT0 00000002:00100000:2.0:1769074932.584861:0:68496:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b03482c4380 x1855007617062528/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/608 e 0 to 0 dl 1769074987 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074932.584875:0:68496:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x240000405:0xa:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074932.584993:0:68496:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074932.585015:0:68496:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074932.585028:0:68496:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.585039:0:68496:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.585049:0:68496:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074932.585054:0:68496:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074932.585058:0:68496:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074932.585063:0:68496:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364241fbf8 00010000:00010000:2.0:1769074932.585066:0:68496:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.585073:0:68496:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.585086:0:68496:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074932.585093:0:68496:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:2.0:1769074932.585102:0:68496:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 44 00000080:00200008:2.0:1769074932.585111:0:68496:0:(file.c:1001:ll_file_open()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000080:00200080:2.0:1769074932.585148:0:68496:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988) cmd=c0086915 arg=20dd3f0 00000100:00100000:2.0:1769074932.585227:0:68496:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68496:1855007617062656:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074932.585579:0:68496:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e6c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.597510:0:68496:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68496:1855007617062656:192.168.204.160@tcp:34:lfs.0 00000080:00200008:2.0:1769074932.597687:0:68496:0:(file.c:424:ll_file_release()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00010000:00010000:2.0:1769074932.597719:0:68496:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074932.622694:0:68498:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074932.622727:0:68498:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074932.622745:0:68498:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074932.622761:0:68498:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074932.622769:0:68498:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074932.622778:0:68498:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074932.622782:0:68498:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074932.622791:0:68498:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642417bf8 00010000:00010000:1.0:1769074932.622795:0:68498:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074932.622804:0:68498:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074932.622818:0:68498:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074932.622824:0:68498:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:1.0:1769074932.622830:0:68498:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:1.0:1769074932.622841:0:68498:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b0344d04a80, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1769074932.622848:0:68498:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open 00000002:00010000:1.0:1769074932.622865:0:68498:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: open flags 0304001 00010000:00010000:1.0:1769074932.622925:0:68498:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68498 initiator: MDT0 00010000:00010000:1.0:1769074932.622932:0:68498:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68498 initiator: MDT0 00000020:00100000:1.0:1769074932.622945:0:68498:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074932.622958:0:68498:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342cead80 x1855007617062912/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074932.622967:0:68498:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68498 initiator: MDT0 00000100:00100000:1.0:1769074932.622990:0:68498:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68498:1855007617062912:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074932.623344:0:68498:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a68480 going to sleep for 50 seconds 00000100:00100000:1.0:1769074932.630085:0:68498:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68498:1855007617062912:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074932.630110:0:68498:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68498 initiator: MDT0 00010000:00010000:1.0:1769074932.630124:0:68498:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68498 initiator: MDT0 00010000:00010000:1.0:1769074932.630130:0:68498:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68498 initiator: MDT0 00010000:00010000:1.0:1769074932.630138:0:68498:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68498 initiator: MDT0 00010000:00010000:1.0:1769074932.630145:0:68498:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68498 initiator: MDT0 00010000:00010000:1.0:1769074932.630152:0:68498:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919c000 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:1.0:1769074932.630157:0:68498:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68498 initiator: MDT0 00010000:00010000:1.0:1769074932.630164:0:68498:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68498 initiator: MDT0 00010000:00010000:1.0:1769074932.630171:0:68498:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 5/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 68498 initiator: MDT0 00010000:00010000:1.0:1769074932.630182:0:68498:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074932.630188:0:68498:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68498 initiator: MDT0 00010000:00010000:1.0:1769074932.630195:0:68498:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c000/0xb7d6bf619879c9f9 lrc: 0/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68498 initiator: MDT0 00000002:00100000:1.0:1769074932.630220:0:68498:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0342cead80 x1855007617062912/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/608 e 0 to 0 dl 1769074987 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074932.630271:0:68498:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x240000405:0xa:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1769074932.630390:0:68498:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074932.630423:0:68498:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074932.630438:0:68498:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074932.630450:0:68498:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074932.630457:0:68498:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074932.630462:0:68498:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074932.630465:0:68498:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074932.630471:0:68498:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642417bf8 00010000:00010000:1.0:1769074932.630474:0:68498:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074932.630481:0:68498:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074932.630495:0:68498:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074932.630503:0:68498:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:1.0:1769074932.630510:0:68498:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 44 00000080:00200008:1.0:1769074932.630517:0:68498:0:(file.c:1001:ll_file_open()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000080:00200080:1.0:1769074932.630548:0:68498:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988) cmd=c0086915 arg=7f73f0 00000100:00100000:1.0:1769074932.630604:0:68498:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68498:1855007617063040:192.168.204.160@tcp:34:lfs.0 00000100:00100000:1.0:1769074932.630679:0:68498:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a68480 going to sleep for 50 seconds 00000100:00100000:1.0:1769074932.646619:0:68498:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68498:1855007617063040:192.168.204.160@tcp:34:lfs.0 00000080:00200008:1.0:1769074932.646837:0:68498:0:(file.c:424:ll_file_release()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00010000:00010000:1.0:1769074932.646871:0:68498:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00000100:00100000:2.0:1769074932.682269:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617063296:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.688788:0:2408:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879ca07 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.688818:0:2408:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879ca07 lrc: 6/2,0 mode: --/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f0d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.688832:0:2408:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879ca07 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f0d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.688844:0:2408:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879ca07 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f0d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074932.688858:0:2408:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879ca07 lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f0d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074932.688871:0:2408:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:2.0:1769074932.688887:0:2408:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879ca07 lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f0d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.688896:0:2408:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879ca07 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f0d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.688902:0:2408:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879ca07 lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f0d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.688909:0:2408:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879ca07 lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f0d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074932.688927:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617063296:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.698759:0:39339:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074932.698767:0:39339:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005bc00/0xb7d6bf619879ca0e lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7fe2323adfd55f1b expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.698775:0:39339:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035005bc00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074932.698779:0:39339:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005bc00/0xb7d6bf619879ca0e lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7fe2323adfd55f1b expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.698786:0:39339:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005bc00/0xb7d6bf619879ca0e lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0x7fe2323adfd55f1b expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.698793:0:39339:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005bc00/0xb7d6bf619879ca0e lrc: 3/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55f1b expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.698807:0:39339:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879ca00 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f06 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.698842:0:39339:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005bc00/0xb7d6bf619879ca0e lrc: 2/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55f1b expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.698849:0:39339:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074932.698872:0:39339:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005bc00/0xb7d6bf619879ca0e lrc: 1/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55f1b expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.698879:0:39339:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005bc00/0xb7d6bf619879ca0e lrc: 0/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55f1b expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.698892:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074932.698894:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074932.698895:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074932.698911:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617063680:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074932.704363:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1acc0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.715294:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617063936:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.715320:0:68501:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PW ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 4/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.715332:0:68501:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 4/1,0 mode: --/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.715344:0:68501:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 4/1,0 mode: PW/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.715353:0:68501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 4/1,0 mode: PW/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.715365:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 5/1,1 mode: PW/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.715372:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 5/1,1 mode: PW/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.715379:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 4/0,1 mode: PW/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074932.715390:0:68501:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=600002b, status=0 req@ffff9b0352a18000 x1855007617063936/t4294968382(4294968382) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2216/1160 e 0 to 0 dl 1769074987 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074932.715404:0:68501:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0352a18000 x1855007617063936/t4294968382(4294968382) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2216/1160 e 0 to 0 dl 1769074987 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074932.715455:0:68501:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 3/0,1 mode: PW/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x81000000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.715468:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 2 res 9663677444/77 (0 0) 00000080:00200000:2.0:1769074932.715475:0:68501:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x240000404:0x4d:0x0] suppgids 0 0: rc 0 00000002:00100000:2.0:1769074932.715495:0:68501:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0352a18000 x1855007617063936/t4294968382(4294968382) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074987 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1769074932.715502:0:68501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca15 00000080:00010000:2.0:1769074932.715508:0:68501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca15 from it ffffb43642407cc0 00010000:00010000:2.0:1769074932.715511:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 3/0,1 mode: PW/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.715530:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 2/0,0 mode: PW/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 5 type: IBT gid 0 flags: 0x91000000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074932.715581:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 8 restore needed 0 00020000:00200000:2.0:1769074932.715594:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074932.715597:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074932.715603:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074932.715614:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074932.715620:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000100:00100000:1.0:1769074932.724597:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074932.724641:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074932.724897:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625281664 00000100:00100000:0.0:1769074932.724919:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625281664 00000100:00100000:0.0:1769074932.724966:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625281792 00000100:00100000:0.0:1769074932.724969:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625281792 00000100:00100000:0.0:1769074932.724984:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8958 00000100:00100000:0.0:1769074932.724992:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8396:x1855007625281664:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074932.725008:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074932.725135:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074932.725142:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cead80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8396:x1855007625281664:12345-192.168.204.160@tcp:104: Request processed in 150us (556us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074932.725149:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8958 00000100:00100000:0.0:1769074932.725173:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8959 00000100:00100000:0.0:1769074932.725179:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342cea680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:36516:x1855007625281792:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074932.725187:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b035005b400/0xb7d6bf619879ca07 lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55f0d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.727058:0:57730:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1769074932.727072:0:57730:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.727084:0:57730:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035005b000 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:2.0:1769074932.727091:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800428400000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074932.727115:0:57730:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800429400000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074932.727129:0:57730:0:(osc_cache.c:1103:osc_extent_make_ready()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800429400000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.728121:0:57730:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (69206016 69210111) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 5/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800429400000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074932.728717:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074932.728732:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342cea680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:36516:x1855007625281792:12345-192.168.204.160@tcp:104: Request processed in 3547us (4085us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074932.728742:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8959 00000008:00100000:2.0:1769074932.729202:0:57730:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b03482c5880 - object 0x2c0000400:56 offset 69206016<>70254592 00000100:00100000:2.0:1769074932.729857:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617064320:192.168.204.160@tcp:4:dd.0 00000100:00100000:2.0:1769074932.762330:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617064064:192.168.204.160@tcp:1:lfs.0 00000008:00010000:2.0:1769074932.814376:0:2408:0:(osc_cache.c:861:osc_extent_finish()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800429400000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074932.814643:0:2408:0:(osc_cache.c:483:__osc_extent_remove()) ### extent: 00000000ae5e0cf9 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x800429400000000 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074932.814681:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617064320:192.168.204.160@tcp:4:dd.0 00000080:00200000:1.0:1769074932.815114:0:57730:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769074932.815669:0:57730:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769074932.816205:0:57730:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 2/0,0 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.816215:0:57730:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1769074932.818467:0:57730:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.818487:0:57730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b035005b000/0xb7d6bf619879c9c8 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 69206016->70254591) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55e11 expref: -99 pid: 68465 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.818516:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074932.818518:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074932.818522:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074932.819291:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617064576:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074932.825120:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617064576:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1769074932.825408:0:68501:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879ca00 lrc: 1/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55f06 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074932.825427:0:68501:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005be00/0xb7d6bf619879c9c1 lrc: 1/0,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0xa/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55e0a expref: -99 pid: 68465 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074932.825433:0:68501:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1769074932.825436:0:68501:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a800/0xb7d6bf619879ca00 lrc: 0/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55f06 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074932.825451:0:68501:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005be00/0xb7d6bf619879c9c1 lrc: 0/0,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0xa/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55e0a expref: -99 pid: 68465 timeout: 0 lvb_type: 3 00000020:00100000:0.0:1769074932.825466:0:68501:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:0.0:1769074932.825475:0:68501:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0352a1a680 x1855007617064832/t0(0) o36->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 472/440 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074932.825498:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617064832:192.168.204.160@tcp:36:lfs.0 00000100:00100000:0.0:1769074932.825614:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b03458543c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.826022:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617064192:192.168.204.160@tcp:1:lfs.0 00000100:00100000:2.0:1769074932.838054:0:68501:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9b0342ad2800: committing for last_committed 4294968383 gen 1 00000100:00100000:2.0:1769074932.838075:0:68501:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff9b0352a1b800 x1855007617063552/t4294968381(4294968381) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074987 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769074932.838093:0:68501:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff9b0352a18000 x1855007617063936/t4294968382(4294968382) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074987 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769074932.838102:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617064832:192.168.204.160@tcp:36:lfs.0 00010000:00010000:2.0:1769074932.838150:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 31 res 9663677444/77 (0 0) 00000080:00010000:2.0:1769074932.838162:0:68501:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000002:00010000:2.0:1769074932.838172:0:68501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4d:0x0]) in obj [0x240000404:0x4d:0x0], intent: layout flags 0 00010000:00010000:2.0:1769074932.838216:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 68501 initiator: MDT0 00010000:00010000:2.0:1769074932.838225:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68501 initiator: MDT0 00010000:00010000:2.0:1769074932.838330:0:68501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT flags: 0x0 pid: 68501 initiator: MDT0 00000100:00100000:2.0:1769074932.838352:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617064960:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074932.838503:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e6c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.865549:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617064960:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.865583:0:68501:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 4/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.865615:0:68501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 4/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000002:00100000:2.0:1769074932.865630:0:68501:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9b03482c6d80 x1855007617064960/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 376/896 e 0 to 0 dl 1769074987 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074932.865640:0:68501:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000002:00010000:2.0:1769074932.865654:0:68501:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.865667:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 16 res 9663677444/77 (0 0) 00000080:00200000:2.0:1769074932.865674:0:68501:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x240000404:0x4d:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1769074932.865692:0:68501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca1c 00000080:00010000:2.0:1769074932.865701:0:68501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca1c from it ffffb43642407c70 00010000:00010000:2.0:1769074932.865705:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.865714:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.865736:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.865745:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074932.865754:0:68501:0:(file.c:6914:ll_layout_lock_set()) ### file [0x240000404:0x4d:0x0](000000005a186648) being reconfigured ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.865797:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.865804:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074932.865822:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074932.865834:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074932.865837:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074932.865843:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074932.865858:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074932.865864:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074932.865883:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1769074932.865895:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.865903:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074932.865910:0:68501:0:(file.c:6914:ll_layout_lock_set()) ### file [0x240000404:0x4d:0x0](000000005a186648) being reconfigured ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074932.865926:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074932.865930:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 2, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074932.865932:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074932.865934:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074932.865937:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074932.865941:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074932.865947:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074932.866021:0:68501:0:(file.c:6809:ll_layout_conf()) [0x240000404:0x4d:0x0]: layout version change: 4294967294 -> 7 00010000:00010000:2.0:1769074932.866026:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.866033:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200080:2.0:1769074932.866058:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=66f4 arg=8ee2a0 00000080:00200008:2.0:1769074932.866071:0:68501:0:(file.c:6252:ll_getattr_dentry()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074932.866092:0:68501:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8),name=f204d.sanity-flr 00000002:00010000:2.0:1769074932.866106:0:68501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4d:0x0]) in obj [0x240000404:0x4d:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074932.866113:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 30 res 9663677444/77 (0 0) 00010000:00010000:2.0:1769074932.866151:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 68501 initiator: MDT0 00010000:00010000:2.0:1769074932.866157:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 68501 initiator: MDT0 00010000:00010000:2.0:1769074932.866167:0:68501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 68501 initiator: MDT0 00000100:00100000:2.0:1769074932.866188:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617065088:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074932.866358:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e6c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.870426:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617065088:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.870453:0:68501:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 4/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.870465:0:68501:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 4/1,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.870475:0:68501:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 4/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.870485:0:68501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 4/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.870498:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 5/2,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.870505:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 5/2,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074932.870513:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 4/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074932.870523:0:68501:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b03482c6d80 x1855007617065088/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 576/1136 e 0 to 0 dl 1769074987 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074932.870534:0:68501:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1769074932.870547:0:68501:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.870561:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 4 res 9663677444/77 (0 0) 00000080:00200000:2.0:1769074932.870567:0:68501:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000404:0x4d:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074932.870631:0:68501:0:(file.c:6809:ll_layout_conf()) [0x240000404:0x4d:0x0]: layout version change: 7 -> 7 00000080:00010000:2.0:1769074932.870637:0:68501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca23 00000080:00010000:2.0:1769074932.870643:0:68501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000080:00010000:2.0:1769074932.870646:0:68501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca23 from it ffffb43642407cd8 00010000:00010000:2.0:1769074932.870649:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.870657:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074932.870687:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:2.0:1769074932.870698:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074932.870701:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074932.870704:0:68501:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240000404:0x4d:0x0] 00000008:00010000:2.0:1769074932.870724:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074932.870727:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e6f0, osc lock ffff9b0345126980, flags 0x400000400001002 00000008:00010000:2.0:1769074932.870735:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074932.870736:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e730, osc lock ffff9b03451265f0, flags 0x400000400001002 00010000:00010000:2.0:1769074932.870780:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 3/1,0 mode: --/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.870788:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 3/1,0 mode: --/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.870798:0:68501:0:(ldlm_request.c:1054:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff9b03482c6d80 x1855007617065216/t0(0) o101->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074932.870846:0:2409:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1769074932.870864:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617065216:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.870979:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.870989:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x400000400001000 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 3/1,0 mode: --/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.870998:0:68501:0:(ldlm_request.c:1054:ldlm_cli_enqueue()) @@@ enqueue lock with no delay req@ffff9b03482c4380 x1855007617065344/t0(0) o101->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 328/400 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000020:00200000:2.0:1769074932.871015:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000100:00100000:2.0:1769074932.871027:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617065344:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.883847:0:2409:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.883875:0:2409:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: --/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.883890:0:2409:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.883901:0:2409:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074932.883918:0:2409:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 70254592, setting rss=70254592; kms=70254592, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/2,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074932.883929:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074932.883934:0:2409:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.883941:0:2409:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.883948:0:2409:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.883956:0:2409:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074932.883975:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617065344:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074932.883993:0:2409:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 6/2,0 mode: --/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.884002:0:2409:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 6/2,0 mode: --/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.884010:0:2409:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.884018:0:2409:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074932.884027:0:2409:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/2,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074932.884035:0:2409:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:2.0:1769074932.884055:0:2409:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.884062:0:2409:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.884069:0:2409:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.884090:0:2409:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074932.884101:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617065216:192.168.204.160@tcp:101:lfs.0 00000080:00200000:0.0:1769074932.884311:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:0.0:1769074932.884318:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_release():245 00000020:00010000:0.0:1769074932.884321:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_cancel():142 00000020:00010000:0.0:1769074932.884326:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e6f0 (ffff9b0342b7d368/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074932.884331:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e730 (ffff9b0342b7de18/1) at cl_lock_cancel():142 00000020:00010000:0.0:1769074932.884333:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b035314e910 (ffff9b0353152a28/0) at cl_lock_fini():72 00000020:00010000:0.0:1769074932.884335:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e6f0 (ffff9b0353152a28/1) at cl_lock_fini():72 00000020:00010000:0.0:1769074932.884341:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e730 (ffff9b0353152a28/1) at cl_lock_fini():72 00000080:00200000:0.0:1769074932.884354:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200080:1.0:1769074932.969897:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=1 00000080:00200008:1.0:1769074932.969919:0:68501:0:(file.c:6252:ll_getattr_dentry()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074932.969927:0:68501:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8),name=f204d.sanity-flr 00000002:00010000:1.0:1769074932.969953:0:68501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4d:0x0]) in obj [0x240000404:0x4d:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074932.969972:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074932.969988:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074932.969996:0:68501:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000404:0x4d:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074932.970004:0:68501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000080:00010000:1.0:1769074932.970007:0:68501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca23 00000080:00010000:1.0:1769074932.970015:0:68501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca23 from it ffffb43642407cd8 00010000:00010000:1.0:1769074932.970019:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074932.970026:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074932.970054:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00020000:00200000:1.0:1769074932.970068:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074932.970072:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:1.0:1769074932.970075:0:68501:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240000404:0x4d:0x0] 00000008:00010000:1.0:1769074932.970098:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074932.970102:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a684b0, osc lock ffff9b0345a08260, flags 0x400000400001002 00000008:00010000:1.0:1769074932.970108:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1769074932.970109:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a684f0, osc lock ffff9b0345a097c0, flags 0x400000400001002 00010000:00010000:1.0:1769074932.970125:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.970135:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074932.970147:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074932.970150:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.970157:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.970177:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.970184:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074932.970190:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1769074932.970192:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.970198:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:1.0:1769074932.970210:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1769074932.970221:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:1.0:1769074932.970226:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:1.0:1769074932.970229:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1769074932.970262:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a684b0 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074932.970266:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a684f0 (ffff9b035121f758/1) at cl_lock_cancel():142 00000020:00010000:1.0:1769074932.970269:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:1.0:1769074932.970272:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a684b0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:1.0:1769074932.970277:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a684f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:1.0:1769074932.970288:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1769074932.970302:0:68501:0:(file.c:5185:ll_file_seek()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0], offset: 10485760, type: SEEK_DATA 00000080:00200000:1.0:1769074932.970307:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:1.0:1769074932.970311:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074932.970314:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1769074932.970318:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1769074932.970323:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1769074932.970327:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1769074932.970341:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000080:00200000:1.0:1769074932.970350:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074932.970355:0:68501:0:(file.c:1759:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 1 00000080:00200000:1.0:1769074932.970357:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074932.970362:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:1.0:1769074932.970364:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:1.0:1769074932.970367:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:1.0:1769074932.970369:0:68501:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [2560, 18446744073709551615] 00000008:00010000:1.0:1769074932.970375:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1769074932.970377:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a684b0, osc lock ffff9b0345a097c0, flags 0x0 00000008:00010000:1.0:1769074932.970380:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:1.0:1769074932.970382:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a684f0, osc lock ffff9b0345a08260, flags 0x0 00010000:00010000:1.0:1769074932.970390:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.970399:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (10485760 67108863) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.970783:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.970794:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 6/2,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.970800:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074932.970814:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.970984:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.970995:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.971003:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.971010:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074932.971021:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074932.977205:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074932.977209:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074932.977216:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a684b0 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074932.977251:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 4/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.977266:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074932.977287:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a684f0 (ffff9b035121f758/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074932.977292:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.977301:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074932.977309:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074932.977312:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a684b0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074932.977316:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a684f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074932.977331:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074932.977342:0:68501:0:(file.c:5216:ll_file_seek()) COMPLETED file f204d.sanity-flr:[0x240000404:0x4d:0x0], offset: 10485760, type: SEEK_DATA, rc = 69206016 00000080:00200008:2.0:1769074932.977353:0:68501:0:(file.c:5185:ll_file_seek()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0], offset: 69206016, type: SEEK_HOLE 00000080:00200000:2.0:1769074932.977359:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:2.0:1769074932.977368:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074932.977371:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074932.977376:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074932.977384:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074932.977399:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000080:00200000:2.0:1769074932.977404:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074932.977419:0:68501:0:(file.c:1759:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 1 00000080:00200000:2.0:1769074932.977422:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074932.977427:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074932.977430:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074932.977432:0:68501:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [16896, 18446744073709551615] 00000008:00010000:2.0:1769074932.977445:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1769074932.977448:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348d6bdb0, osc lock ffff9b0345127c80, flags 0x0 00010000:00010000:2.0:1769074932.977465:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.977477:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (69206016 134217727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.977489:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.977497:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.977504:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074932.977513:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074932.983306:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074932.983312:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074932.983317:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6bdb0 (ffff9b035121f758/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074932.983327:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.983341:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074932.983357:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074932.983360:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6bdb0 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074932.983378:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074932.983389:0:68501:0:(file.c:5216:ll_file_seek()) COMPLETED file f204d.sanity-flr:[0x240000404:0x4d:0x0], offset: 69206016, type: SEEK_HOLE, rc = 70254592 00000080:00200080:2.0:1769074932.983404:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=0 00000080:00200080:2.0:1769074932.983423:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=2 00000080:00200008:2.0:1769074932.983429:0:68501:0:(file.c:6252:ll_getattr_dentry()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074932.983435:0:68501:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8),name=f204d.sanity-flr 00000002:00010000:2.0:1769074932.983455:0:68501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4d:0x0]) in obj [0x240000404:0x4d:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074932.983470:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.983481:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074932.983488:0:68501:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000404:0x4d:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074932.983494:0:68501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000080:00010000:2.0:1769074932.983497:0:68501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca23 00000080:00010000:2.0:1769074932.983503:0:68501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca23 from it ffffb43642407cd8 00010000:00010000:2.0:1769074932.983506:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.983512:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074932.983532:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074932.983543:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074932.983546:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074932.983550:0:68501:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240000404:0x4d:0x0] 00000008:00010000:2.0:1769074932.983566:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074932.983571:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e3f0, osc lock ffff9b0345127c80, flags 0x400000400001002 00000008:00010000:2.0:1769074932.983578:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074932.983580:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e430, osc lock ffff9b03451264c0, flags 0x400000400001002 00010000:00010000:2.0:1769074932.983853:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.983864:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074932.983877:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074932.983881:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.983888:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.983903:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.983912:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074932.983919:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074932.983922:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.983929:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074932.983941:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074932.983952:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074932.983955:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074932.983958:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074932.983960:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e3f0 (ffff9b035121f758/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074932.983963:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e430 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074932.983966:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074932.983968:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e3f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074932.983971:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e430 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074932.983978:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074932.984017:0:68501:0:(llite_lib.c:2314:ll_setattr_raw()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8) current size 70254592, valid attrs 2068, mode 0, uid -1802657998, gid -1, new size 10485760, atime 1769074932.000000000, mtime 1769074932.000000000, ctime 1769074932.000000000 00010000:00010000:2.0:1769074932.984040:0:68501:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.984056:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.989498:0:68501:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 1/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074932.989518:0:68501:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074932.989522:0:68501:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca23 lrc: 0/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55f61 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000020:00100000:2.0:1769074932.989550:0:68501:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074932.989564:0:68501:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7480 x1855007617065856/t0(0) o36->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074932.990150:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617065856:192.168.204.160@tcp:36:lfs.0 00000100:00100000:2.0:1769074932.990815:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074932.998729:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617065856:192.168.204.160@tcp:36:lfs.0 00000080:00200000:2.0:1769074932.998775:0:68501:0:(file.c:1759:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 2 00000080:00200000:2.0:1769074932.998783:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:2.0:1769074932.998804:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x11 00000080:00200000:2.0:1769074932.998821:0:68501:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1769074932.998839:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1769074932.998843:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e3f0, osc lock ffff9b03451264c0, flags 0x0 00000008:00010000:2.0:1769074932.998847:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1769074932.998849:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e430, osc lock ffff9b0345127c80, flags 0x0 00010000:00010000:2.0:1769074932.998909:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.998921:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074932.998931:0:68501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->10485759] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074932.998951:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617065984:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074932.999030:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ea80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074933.002927:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617065984:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074933.002955:0:68501:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.002972:0:68501:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.002984:0:68501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.002997:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074933.003008:0:68501:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.003018:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.003025:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.003112:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 3/0,1 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.003119:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 3/0,1 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [10485760->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.003128:0:68501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 3/0,1 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [10485760->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074933.003149:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617066112:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074933.003342:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ea80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074933.009158:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617066112:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074933.009189:0:68501:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 4/0,1 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.009208:0:68501:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.009221:0:68501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.009383:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074933.009397:0:68501:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.009513:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.009521:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074933.012372:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074933.012380:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.012384:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.012388:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e3f0 (ffff9b035121f758/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074933.012491:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.012506:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074933.012522:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e430 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074933.012527:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.012535:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074933.012543:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.012546:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e3f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.012550:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e430 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.012567:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074933.012579:0:68501:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8) current size 70254592, valid attrs 2168, mode 0, uid -1802657998, gid -1, new size 10485760, atime 1769074932.000000000, mtime 1769074932.000000000, ctime 1769074932.000000000, rc 0 00000080:00200080:2.0:1769074933.012681:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=0 00000080:00200080:2.0:1769074933.012693:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=1 00000080:00200008:2.0:1769074933.012699:0:68501:0:(file.c:6252:ll_getattr_dentry()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074933.012706:0:68501:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8),name=f204d.sanity-flr 00000002:00010000:2.0:1769074933.012726:0:68501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4d:0x0]) in obj [0x240000404:0x4d:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074933.012739:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 30 res 9663677444/77 (0 0) 00010000:00010000:2.0:1769074933.012865:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x0/0x0 rrc: 5 type: IBT flags: 0x10000000000000 pid: 68501 initiator: MDT0 00010000:00010000:2.0:1769074933.012874:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 68501 initiator: MDT0 00010000:00010000:2.0:1769074933.012883:0:68501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT flags: 0x0 pid: 68501 initiator: MDT0 00000100:00100000:2.0:1769074933.012901:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617066368:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074933.013021:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074933.017178:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617066368:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074933.017207:0:68501:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 4/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x2/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.017219:0:68501:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 4/1,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.017265:0:68501:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 4/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.017275:0:68501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 4/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.017287:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 5/2,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.017293:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 5/2,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.017300:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 4/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074933.017310:0:68501:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b03482c4a80 x1855007617066368/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 576/1136 e 0 to 0 dl 1769074988 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074933.017321:0:68501:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1769074933.017335:0:68501:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.017348:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 4 res 9663677444/77 (0 0) 00000080:00200000:2.0:1769074933.017355:0:68501:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000404:0x4d:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074933.017439:0:68501:0:(file.c:6809:ll_layout_conf()) [0x240000404:0x4d:0x0]: layout version change: 7 -> 7 00000080:00010000:2.0:1769074933.017444:0:68501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca46 00000080:00010000:2.0:1769074933.017449:0:68501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000080:00010000:2.0:1769074933.017453:0:68501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca46 from it ffffb43642407cd8 00010000:00010000:2.0:1769074933.017456:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.017463:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074933.017494:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074933.017506:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074933.017509:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074933.017513:0:68501:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240000404:0x4d:0x0] 00000008:00010000:2.0:1769074933.017532:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074933.017536:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e3f0, osc lock ffff9b0345127c80, flags 0x400000400001002 00000008:00010000:2.0:1769074933.017546:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074933.017548:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e430, osc lock ffff9b03451264c0, flags 0x400000400001002 00010000:00010000:2.0:1769074933.017565:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.017576:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.017586:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074933.017608:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.017616:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.017631:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.017640:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.017647:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074933.017650:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.017656:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.017664:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074933.017674:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074933.017679:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.017682:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.017685:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e3f0 (ffff9b035121f758/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.017689:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e430 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.017692:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.017695:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e3f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.017698:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e430 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.017708:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074933.017738:0:68501:0:(file.c:2474:do_file_read_iter()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0], ppos: 69206016, count: 1048576 00000080:00200000:2.0:1769074933.017744:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:2.0:1769074933.017748:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074933.017751:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00000080:00200000:2.0:1769074933.017756:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074933.017764:0:68501:0:(file.c:1994:ll_file_io_generic()) f204d.sanity-flr: read ppos: 69206016, bytes: 1048576 00000080:00200000:2.0:1769074933.017772:0:68501:0:(file.c:1759:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 1 00000020:00200000:2.0:1769074933.017780:0:68501:0:(cl_io.c:214:cl_io_rw_init()) header@000000002257c179[0x4, 2, [0x240000404:0x4d:0x0] hash] 00000020:00200000:2.0:1769074933.017782:0:68501:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [69206016, 70254592) 2048 0 00000080:00200000:2.0:1769074933.017784:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:2.0:1769074933.017791:0:68501:0:(file.c:2072:ll_file_io_generic()) Range lock [16896, 17151] 00020000:00200000:2.0:1769074933.017798:0:68501:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 69206016 chunk: [69206016, 70254592) 70254592, 1048576 00020000:00200000:2.0:1769074933.017803:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00000080:00200000:2.0:1769074933.017815:0:68501:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [16896, 17151] 00000008:00010000:2.0:1769074933.017823:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:2.0:1769074933.017826:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348d6bdb0, osc lock ffff9b03451264c0, flags 0x40000 00000080:00200000:2.0:1769074933.017831:0:68501:0:(vvp_io.c:827:vvp_io_read_start()) f204d.sanity-flr: read [69206016, 70254592) 00000080:00200000:2.0:1769074933.017841:0:68501:0:(vvp_io.c:886:vvp_io_read_start()) count:1048576 iocb pos:69206016 00000080:00200000:2.0:1769074933.017849:0:68501:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8), size=1048576 (max 603979776), offset=69206016=0x4200000, pages 256 (max 147456), lockless, parallel 00000020:00200000:2.0:1769074933.018156:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1769074933.018161:0:68501:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 1048576 count 1048576 file_offset 70254592 00000020:00010000:2.0:1769074933.018169:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.018171:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.018175:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6bdb0 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.018177:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.018180:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6bdb0 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.018185:0:68501:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1769074933.018191:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1769074933.018218:0:57084:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b0342c54840 00000008:00100000:2.0:1769074933.018329:0:57084:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b03482c4a80 - object 0x2c0000400:56 offset 69206016<>70254592 00000080:00200000:2.0:1769074933.046264:0:68501:0:(file.c:2109:ll_file_io_generic()) Range unlock [16896, 17151] 00000080:00200000:2.0:1769074933.046281:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074933.046286:0:68501:0:(file.c:2142:ll_file_io_generic()) f204d.sanity-flr: 1 io complete with rc: 0, result: 1048576, restart: 0 00000020:00200000:2.0:1769074933.046289:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074933.046298:0:68501:0:(file.c:2228:ll_file_io_generic()) iot: 1, result: 1048576 00000080:00200080:2.0:1769074933.046317:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=0 00000080:00200080:2.0:1769074933.046327:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=2 00000080:00200008:2.0:1769074933.046333:0:68501:0:(file.c:6252:ll_getattr_dentry()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074933.046339:0:68501:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8),name=f204d.sanity-flr 00000002:00010000:2.0:1769074933.046362:0:68501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4d:0x0]) in obj [0x240000404:0x4d:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074933.046380:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.046394:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074933.046401:0:68501:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000404:0x4d:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074933.046420:0:68501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000080:00010000:2.0:1769074933.046424:0:68501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca46 00000080:00010000:2.0:1769074933.046433:0:68501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca46 from it ffffb43642407cd8 00010000:00010000:2.0:1769074933.046436:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.046443:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074933.046466:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074933.046477:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074933.046480:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074933.046484:0:68501:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240000404:0x4d:0x0] 00000008:00010000:2.0:1769074933.046502:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074933.046506:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e3f0, osc lock ffff9b03451264c0, flags 0x400000400001002 00000008:00010000:2.0:1769074933.046512:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074933.046513:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e430, osc lock ffff9b0345127c80, flags 0x400000400001002 00010000:00010000:2.0:1769074933.046528:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.046538:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.046547:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074933.046550:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.046557:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.046571:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.046580:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.046587:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074933.046590:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.046596:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.046611:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074933.046621:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074933.046625:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.046628:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.046630:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e3f0 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.046635:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e430 (ffff9b035121f758/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.046637:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.046640:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e3f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.046644:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e430 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.046651:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074933.046671:0:68501:0:(file.c:2648:do_file_write_iter()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0], ppos: 69206016, count: 1048576 00000080:00200000:2.0:1769074933.046678:0:68501:0:(file.c:1994:ll_file_io_generic()) f204d.sanity-flr: write ppos: 69206016, bytes: 1048576 00000080:00200000:2.0:1769074933.046685:0:68501:0:(file.c:1759:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 2 00000020:00200000:2.0:1769074933.046693:0:68501:0:(cl_io.c:214:cl_io_rw_init()) header@000000002257c179[0x4, 2, [0x240000404:0x4d:0x0] hash] 00000020:00200000:2.0:1769074933.046695:0:68501:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [69206016, 70254592) 2048 0 00000080:00200000:2.0:1769074933.046698:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00000080:00200000:2.0:1769074933.046708:0:68501:0:(file.c:2072:ll_file_io_generic()) Range lock [16896, 17151] 00020000:00200000:2.0:1769074933.046715:0:68501:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 69206016 chunk: [69206016, 70254592) 70254592, 1048576 00020000:00200000:2.0:1769074933.046719:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x11 00000080:00200000:2.0:1769074933.046730:0:68501:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [16896, 17151] 00000008:00010000:2.0:1769074933.046740:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 11 00000008:00010000:2.0:1769074933.046743:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348d6bdb0, osc lock ffff9b0345127c80, flags 0x40000 00000080:00200000:2.0:1769074933.046750:0:68501:0:(vvp_io.c:1312:vvp_io_write_start()) f204d.sanity-flr: write [69206016, 70254592) append 0 append lockpos 0 00000080:00200000:2.0:1769074933.046783:0:68501:0:(rw26.c:439:ll_direct_IO_impl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8), size=1048576 (max 603979776), offset=69206016=0x4200000, pages 256 (max 147456), lockless, parallel 00000020:00200000:2.0:1769074933.047167:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1769074933.047170:0:68501:0:(rw26.c:570:ll_direct_IO_impl()) result 0 tot_bytes 1048576 count 1048576 file_offset 70254592 00000080:00200000:2.0:1769074933.047181:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074933.047185:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.047188:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.047190:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6bdb0 (ffff9b035121f758/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.047193:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.047195:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6bdb0 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.047200:0:68501:0:(vvp_io.c:518:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1769074933.047205:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1769074933.047226:0:57084:0:(osc_cache.c:1892:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff9b0342c54d80 00000008:00100000:2.0:1769074933.047758:0:57084:0:(osc_request.c:1967:osc_brw_prep_request()) brw rpc ffff9b03482c5500 - object 0x280000400:55 offset 69206016<>70254592 00000100:00100000:0.0:1769074933.051193:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074933.051388:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625282816 00000100:00100000:1.0:1769074933.051399:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625282816 00000100:00100000:1.0:1769074933.051481:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8960 00000100:00100000:1.0:1769074933.051491:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a38380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28434:x1855007625282816:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074933.051508:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074933.051575:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074933.051581:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a38380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28434:x1855007625282816:12345-192.168.204.160@tcp:104: Request processed in 93us (399us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074933.051589:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8960 00010000:00010000:2.0:1769074933.052073:0:60220:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074933.052082:0:60220:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.052091:0:60220:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860a00 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:2.0:1769074933.052097:0:60220:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800428400000000 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074933.052118:0:60220:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074933.052133:0:60220:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769074933.052176:0:60220:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 2/0,0 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.052183:0:60220:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074933.052214:0:60220:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 1/0,0 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.052221:0:60220:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca3f lrc: 0/0,0 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55f84 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.053529:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074933.053532:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074933.053534:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1769074933.098731:0:68501:0:(file.c:2109:ll_file_io_generic()) Range unlock [16896, 17151] 00000080:00200000:2.0:1769074933.098747:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074933.098752:0:68501:0:(file.c:2142:ll_file_io_generic()) f204d.sanity-flr: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1769074933.098765:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00200000:2.0:1769074933.098769:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074933.098777:0:68501:0:(file.c:2228:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200080:2.0:1769074933.098794:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=0 00000080:00200080:2.0:1769074933.098801:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=1 00000080:00200008:2.0:1769074933.098807:0:68501:0:(file.c:6252:ll_getattr_dentry()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074933.098813:0:68501:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8),name=f204d.sanity-flr 00000002:00010000:2.0:1769074933.098835:0:68501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4d:0x0]) in obj [0x240000404:0x4d:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074933.098852:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.098870:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074933.098877:0:68501:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000404:0x4d:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074933.098885:0:68501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000080:00010000:2.0:1769074933.098888:0:68501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca46 00000080:00010000:2.0:1769074933.098895:0:68501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca46 from it ffffb43642407cd8 00010000:00010000:2.0:1769074933.098899:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.098907:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074933.098928:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074933.098937:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074933.098941:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074933.098944:0:68501:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240000404:0x4d:0x0] 00000008:00010000:2.0:1769074933.098963:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074933.098967:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e3f0, osc lock ffff9b0345127c80, flags 0x400000400001002 00000008:00010000:2.0:1769074933.098973:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074933.098974:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e430, osc lock ffff9b03451264c0, flags 0x400000400001002 00010000:00010000:2.0:1769074933.098989:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.098998:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.099008:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074933.099011:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099018:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099034:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099042:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.099049:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074933.099052:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099059:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.099067:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074933.099072:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074933.099076:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.099078:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.099081:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e3f0 (ffff9b035121f758/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.099084:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e430 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.099087:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.099089:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e3f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.099092:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e430 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.099100:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074933.099110:0:68501:0:(file.c:5185:ll_file_seek()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0], offset: 70254592, type: SEEK_DATA 00000080:00200000:2.0:1769074933.099115:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:2.0:1769074933.099119:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074933.099121:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074933.099125:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074933.099130:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074933.099142:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000080:00200000:2.0:1769074933.099150:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1769074933.099155:0:68501:0:(file.c:1759:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 1 00000080:00200000:2.0:1769074933.099157:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074933.099161:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074933.099165:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074933.099167:0:68501:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 0 [17152, 18446744073709551615] 00000008:00010000:2.0:1769074933.099174:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 8 00000008:00010000:2.0:1769074933.099176:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0348d6bdb0, osc lock ffff9b03451264c0, flags 0x0 00010000:00010000:2.0:1769074933.099183:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099191:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (70254592 134217727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099198:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099282:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 6/2,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099290:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074933.099301:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074933.099306:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.099308:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.099311:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0348d6bdb0 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074933.099314:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099320:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 3/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074933.099327:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.099330:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0348d6bdb0 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.099337:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074933.099344:0:68501:0:(file.c:5185:ll_file_seek()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0], offset: 0, type: (null) 00000080:00200000:2.0:1769074933.099348:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074933.099352:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074933.099354:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074933.099355:0:68501:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240000404:0x4d:0x0] 00000008:00010000:2.0:1769074933.099362:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074933.099364:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e3f0, osc lock ffff9b03451264c0, flags 0x400000400001002 00000008:00010000:2.0:1769074933.099367:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074933.099369:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e430, osc lock ffff9b0345127c80, flags 0x400000400001002 00010000:00010000:2.0:1769074933.099376:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099385:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.099393:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074933.099396:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099402:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099422:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099431:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.099438:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074933.099440:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.099447:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.099454:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074933.099458:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074933.099460:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.099462:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.099465:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e3f0 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.099467:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e430 (ffff9b035121f758/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.099470:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.099472:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e3f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.099474:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e430 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.099480:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074933.099486:0:68501:0:(file.c:5216:ll_file_seek()) COMPLETED file f204d.sanity-flr:[0x240000404:0x4d:0x0], offset: 0, type: (null), rc = 70254592 00000080:00200080:2.0:1769074933.099491:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=0 00000080:00200080:2.0:1769074933.111393:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=2 00000080:00200008:2.0:1769074933.111428:0:68501:0:(file.c:6252:ll_getattr_dentry()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1769074933.111436:0:68501:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8),name=f204d.sanity-flr 00000002:00010000:2.0:1769074933.111461:0:68501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4d:0x0]) in obj [0x240000404:0x4d:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074933.111481:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.111495:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074933.111503:0:68501:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000404:0x4d:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074933.111512:0:68501:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000080:00010000:2.0:1769074933.111515:0:68501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca46 00000080:00010000:2.0:1769074933.111523:0:68501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca46 from it ffffb43642407cd8 00010000:00010000:2.0:1769074933.111529:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.111538:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074933.111574:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 restore needed 0 00020000:00200000:2.0:1769074933.111587:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074933.111591:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use non-delayed RPC state for this IO 00000001:00010000:2.0:1769074933.111596:0:68501:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x240000404:0x4d:0x0] 00000008:00010000:2.0:1769074933.111755:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074933.111759:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e3f0, osc lock ffff9b0345127c80, flags 0x400000400001002 00000008:00010000:2.0:1769074933.111766:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1769074933.111768:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e430, osc lock ffff9b03451264c0, flags 0x400000400001002 00010000:00010000:2.0:1769074933.111784:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.111795:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 67108863) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.111806:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074933.111809:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 5/1,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.111816:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 4/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.111832:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.111841:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (67108864 134217727) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.111848:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1769074933.111850:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 5/1,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.111863:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 4/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00200000:2.0:1769074933.111870:0:68501:0:(cl_io.c:1802:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1769074933.111883:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074933.111889:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.111892:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.111895:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e3f0 (ffff9b035121f758/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.111899:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e430 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.111902:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0356960db0 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.111905:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e3f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.111910:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e430 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.111920:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/1, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074933.111939:0:68501:0:(llite_lib.c:2314:ll_setattr_raw()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8) current size 70254592, valid attrs 2068, mode 0, uid -1802657998, gid -1, new size 70254592, atime 1769074933.000000000, mtime 1769074933.000000000, ctime 1769074933.000000000 00010000:00010000:2.0:1769074933.111961:0:68501:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.111978:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.112023:0:68501:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 1/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.112030:0:68501:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074933.112033:0:68501:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca46 lrc: 0/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55f92 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000020:00100000:2.0:1769074933.112058:0:68501:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074933.112071:0:68501:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c4000 x1855007617067008/t0(0) o36->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074933.112094:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617067008:192.168.204.160@tcp:36:lfs.0 00000100:00100000:2.0:1769074933.112329:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074933.118699:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617067008:192.168.204.160@tcp:36:lfs.0 00000080:00200000:2.0:1769074933.118849:0:68501:0:(file.c:1759:ll_io_set_mirror()) f204d.sanity-flr: desiginated mirror: 2 00000080:00200000:2.0:1769074933.118856:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:2.0:1769074933.118879:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x11 00000080:00200000:2.0:1769074933.118895:0:68501:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1769074933.118913:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1769074933.118918:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e3f0, osc lock ffff9b03451264c0, flags 0x0 00000008:00010000:2.0:1769074933.118922:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1769074933.118924:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e430, osc lock ffff9b0345127c80, flags 0x0 00010000:00010000:2.0:1769074933.118945:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 2/0,1 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.118959:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 10485759) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 2/0,1 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.118972:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 6/0,2 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.118980:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 6/0,2 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.118992:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.119040:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 3/0,1 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.119048:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 3/0,1 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [10485760->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.119058:0:68501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 3/0,1 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [10485760->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074933.119078:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617067136:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074933.119227:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ea80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074933.123695:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617067136:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074933.123723:0:68501:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 4/0,1 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.123739:0:68501:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.123838:0:68501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.123855:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074933.123869:0:68501:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 70254592, setting rss=70254592; kms=70254592, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.123881:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.123887:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074933.132679:0:68501:0:(file.c:1686:ll_merge_attr_nolock()) [0x240000404:0x4d:0x0] updating i_size 70254592 i_blocks 2048 00000020:00010000:2.0:1769074933.132688:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.132692:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.132697:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e3f0 (ffff9b035121f758/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074933.132707:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.132731:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074933.132747:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e430 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074933.132758:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x810020000000000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.132766:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074933.132780:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.132783:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e3f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.132788:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e430 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.132807:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074933.132818:0:68501:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8) current size 70254592, valid attrs 2168, mode 0, uid -1802657998, gid -1, new size 70254592, atime 1769074933.000000000, mtime 1769074933.000000000, ctime 1769074933.000000000, rc 0 00000080:00200080:2.0:1769074933.132837:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=400866b1 arg=0 00000080:00200008:2.0:1769074933.132863:0:68501:0:(llite_lib.c:2314:ll_setattr_raw()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8) current size 70254592, valid attrs 101f0, mode b436, uid 1618691992, gid -25853, new size 18446633037359620552, atime 1769074907.000000000, mtime 1769074919.000000000, ctime 1769074933.000000000 00000020:00100000:2.0:1769074933.132934:0:68501:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1769074933.132943:0:68501:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c6680 x1855007617067520/t0(0) o36->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769074933.132962:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617067520:192.168.204.160@tcp:36:lfs.0 00000100:00100000:2.0:1769074933.133041:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074933.141965:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617067520:192.168.204.160@tcp:36:lfs.0 00000080:00200000:2.0:1769074933.142176:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 restore needed 0 00020000:00200000:2.0:1769074933.142191:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074933.142195:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074933.142203:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074933.142216:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074933.142225:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074933.142228:0:68501:0:(vvp_io.c:199:vvp_io_one_lock_index()) lock: 1 [0, 18446744073709551615] 00000008:00010000:2.0:1769074933.142327:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1769074933.142332:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e3f0, osc lock ffff9b0345127c80, flags 0x0 00000008:00010000:2.0:1769074933.142336:0:68501:0:(lustre_osc.h:163:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1769074933.142338:0:68501:0:(osc_lock.c:1248:osc_lock_init()) ### lock ffff9b0352a3e430, osc lock ffff9b03451264c0, flags 0x0 00010000:00010000:2.0:1769074933.142354:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b034921d000 type 11 mode 2 res 10737419264/54 (0 67108863) 00010000:00010000:2.0:1769074933.142396:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.142417:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.142427:0:68501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 3/0,1 mode: --/PW res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074933.142591:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617067648:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074933.142732:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ea80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074933.147524:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074933.147609:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625282944 00000100:00100000:0.0:1769074933.147620:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625282944 00000100:00100000:0.0:1769074933.147680:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8961 00000100:00100000:0.0:1769074933.147691:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ceb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8398:x1855007625282944:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074933.147708:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.147798:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074933.147805:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ceb100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8398:x1855007625282944:12345-192.168.204.160@tcp:104: Request processed in 117us (373us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074933.147813:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8961 00010000:00010000:2.0:1769074933.147943:0:46917:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074933.147948:0:46917:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.147955:0:46917:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365861e00 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:2.0:1769074933.147960:0:46917:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 3/0,0 mode: PR/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800428400000000 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074933.147975:0:46917:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1769074933.147998:0:46917:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:2.0:1769074933.148047:0:46917:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 2/0,0 mode: --/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.148054:0:46917:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074933.148080:0:46917:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 1/0,0 mode: --/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.148089:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca2a lrc: 0/0,0 mode: --/PR res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x804c69400000020 nid: local remote: 0x7fe2323adfd55f68 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.148102:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074933.148103:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074933.148104:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074933.166769:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074933.166892:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625283072 00000100:00100000:0.0:1769074933.166903:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625283072 00000100:00100000:0.0:1769074933.166958:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8962 00000100:00100000:0.0:1769074933.166969:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6500:x1855007625283072:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074933.166985:0:39330:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.166997:0:39330:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->67108863] (req 0->67108863) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.167005:0:39330:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 4/0,1 mode: --/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.167015:0:39330:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.167030:0:39330:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.167039:0:39330:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 5/0,1 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x80000000000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.167059:0:39330:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b0365861000) 00000100:00100000:0.0:1769074933.167116:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce8e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6500:x1855007625283072:12345-192.168.204.160@tcp:105: Request processed in 150us (368us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074933.167123:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8962 00000100:00100000:2.0:1769074933.169179:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617067648:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074933.169206:0:68501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.169318:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10000000020000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074933.169332:0:68501:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 4/0,2 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x10020000020000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.169345:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 5/0,2 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.169353:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.169388:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0343b61800 type 11 mode 2 res 11811161088/56 (67108864 134217727) 00010000:00010000:2.0:1769074933.169440:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.169454:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.169464:0:68501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 3/0,1 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000100:00100000:2.0:1769074933.169487:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617067904:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074933.169701:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ea80 going to sleep for 11 seconds 00000100:00100000:1.0:1769074933.190266:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074933.190357:0:55997:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074933.190376:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625283200 00000100:00100000:0.0:1769074933.190385:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625283200 00000100:00100000:0.0:1769074933.190452:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8963 00000100:00100000:0.0:1769074933.190462:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b03513cb050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8398:x1855007625283200:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074933.190478:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879ca31 lrc: 2/0,0 mode: PR/PR res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800420000000000 nid: local remote: 0x7fe2323adfd55f6f expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.190547:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074933.190554:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b03513cb050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8398:x1855007625283200:12345-192.168.204.160@tcp:104: Request processed in 94us (338us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074933.190560:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8963 00000100:00100000:1.0:1769074933.205676:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074933.205769:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625283328 00000100:00100000:0.0:1769074933.205780:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625283328 00000100:00100000:0.0:1769074933.205842:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8964 00000100:00100000:0.0:1769074933.205851:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6500:x1855007625283328:12345-192.168.204.160@tcp:105: 00010000:00010000:0.0:1769074933.205867:0:39330:0:(ldlm_lockd.c:2517:ldlm_callback_handler()) ### completion ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.205877:0:39330:0:(ldlm_lockd.c:1967:ldlm_handle_cp_callback()) ### client completion callback handler START ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [67108864->134217727] (req 67108864->134217727) gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.205886:0:39330:0:(ldlm_lockd.c:2044:ldlm_handle_cp_callback()) ### completion AST, new policy data ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 4/0,1 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.205895:0:39330:0:(ldlm_lock.c:709:ldlm_add_cp_work_item()) ### lock granted; sending completion AST. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.205903:0:39330:0:(ldlm_resource.c:1525:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10080000000000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.205912:0:39330:0:(ldlm_lockd.c:2072:ldlm_handle_cp_callback()) ### callback handler finished, about to run_ast_work ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 5/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x80000000000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074933.205930:0:39330:0:(ldlm_lockd.c:2083:ldlm_handle_cp_callback()) ### client completion callback handler END (lock ffff9b0365861e00) 00000100:00100000:0.0:1769074933.205984:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:6500:x1855007625283328:12345-192.168.204.160@tcp:105: Request processed in 134us (314us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074933.205991:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8964 00000100:00100000:2.0:1769074933.206661:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617067904:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074933.206683:0:68501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x20000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.206698:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10000000020000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074933.206709:0:68501:0:(osc_lock.c:179:osc_lock_lvb_update()) ### acquired size 70254592, setting rss=70254592; kms=70254592, end=18446744073709551615 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 4/0,2 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x10020000020000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.206721:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 5/0,2 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.206729:0:68501:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074933.219388:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_release():245 00000020:00010000:2.0:1769074933.219393:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_cancel():142 00000020:00010000:2.0:1769074933.219399:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e3f0 (ffff9b035121f758/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074933.219418:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 4/0,1 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.219430:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074933.219446:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9b0352a3e430 (ffff9b0342ca8cf8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1769074933.219451:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 4/0,1 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x810020000020000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:2.0:1769074933.219459:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000020:00010000:2.0:1769074933.219467:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0359283308 (ffff9b035121f908/0) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.219469:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e3f0 (ffff9b035121f908/1) at cl_lock_fini():72 00000020:00010000:2.0:1769074933.219474:0:68501:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9b0352a3e430 (ffff9b035121f908/1) at cl_lock_fini():72 00000080:00200000:2.0:1769074933.219493:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 0/0, layout version 7 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1769074933.219506:0:68501:0:(llite_lib.c:2564:ll_setattr_raw()) COMPLETED file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8) current size 70254592, valid attrs 101f0, mode b436, uid 1618691992, gid -25853, new size 18446633037359620552, atime 1769074907.000000000, mtime 1769074919.000000000, ctime 1769074933.000000000, rc 0 00000080:00200080:2.0:1769074933.219528:0:68501:0:(file.c:4617:ll_file_ioctl()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) cmd=c00c66f3 arg=8ee2a0 00010000:00010000:2.0:1769074933.219581:0:68501:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 2/0,0 mode: PW/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x81008000000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.219596:0:68501:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005a400/0xb7d6bf619879ca15 lrc: 0/0,0 mode: --/PW res: [0x240000404:0x4d:0x0].0x0 bits 0x4/0x0 rrc: 4 type: IBT gid 0 flags: 0x85809400000000 nid: local remote: 0x7fe2323adfd55f30 expref: -99 pid: 68501 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074933.219619:0:68501:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0352a18000 x1855007617063936/t4294968382(4294968382) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074987 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074933.219629:0:68501:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074933.219634:0:68501:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7100 x1855007617068544/t0(0) o35->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 504/2192 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074933.219655:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617068544:192.168.204.160@tcp:35:lfs.0 00000100:00100000:2.0:1769074933.221049:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 11 seconds 00000100:00100000:1.0:1769074933.226559:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074933.226658:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625283456 00000100:00100000:0.0:1769074933.226669:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625283456 00000100:00100000:0.0:1769074933.226727:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8965 00000100:00100000:0.0:1769074933.226736:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:15947:x1855007625283456:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074933.226766:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074933.226841:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074933.226848:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:15947:x1855007625283456:12345-192.168.204.160@tcp:104: Request processed in 114us (300us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074933.226856:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8965 00010000:00010000:2.0:1769074933.227003:0:39339:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769074933.227010:0:39339:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.227018:0:39339:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860400 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074933.227023:0:39339:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.227031:0:39339:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.227039:0:39339:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 3/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074933.227064:0:39339:0:(vvp_object.c:114:vvp_conf_set()) [0x240000404:0x4d:0x0]: losing layout lock 00010000:00010000:2.0:1769074933.227102:0:39339:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 2/0,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.227109:0:39339:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1769074933.227139:0:39339:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 1/0,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.227147:0:39339:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca1c lrc: 0/0,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x8 rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd55f53 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.227163:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074933.227164:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074933.227166:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074933.236486:0:68501:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9b0342ad2800: committing for last_committed 4294968389 gen 1 00000100:00100000:2.0:1769074933.236522:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617068544:192.168.204.160@tcp:35:lfs.0 00000002:00100000:2.0:1769074933.236554:0:68501:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0352a18000 x1855007617063936/t4294968382(4294968382) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074987 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00010000:2.0:1769074933.236587:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 31 res 9663677444/77 (0 0) 00000080:00010000:2.0:1769074933.236598:0:68501:0:(file.c:7031:ll_layout_intent()) ### lustre: requeue layout lock for file [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000002:00010000:2.0:1769074933.236608:0:68501:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4d:0x0]) in obj [0x240000404:0x4d:0x0], intent: layout flags 0 00010000:00010000:2.0:1769074933.236648:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 68501 initiator: MDT0 00010000:00010000:2.0:1769074933.236656:0:68501:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68501 initiator: MDT0 00010000:00010000:2.0:1769074933.236664:0:68501:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 3/1,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 68501 initiator: MDT0 00000100:00100000:2.0:1769074933.236682:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617068800:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074933.236757:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074933.244581:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617068800:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074933.244613:0:68501:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 4/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.244628:0:68501:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 4/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000002:00100000:2.0:1769074933.244644:0:68501:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=400 disposition=1, status=0 req@ffff9b03482c7100 x1855007617068800/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 376/896 e 0 to 0 dl 1769074988 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074933.244655:0:68501:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: layout, lvb_len: 552 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000002:00010000:2.0:1769074933.244669:0:68501:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.244681:0:68501:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 16 res 9663677444/77 (0 0) 00000080:00200000:2.0:1769074933.244687:0:68501:0:(namei.c:955:ll_intent_lock()) intent lock 1024 on i1 [0x240000404:0x4d:0x0] suppgids 0 0: rc 0 00000080:00010000:2.0:1769074933.244704:0:68501:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca62 00000080:00010000:2.0:1769074933.244713:0:68501:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca62 from it ffffb43642407bc8 00010000:00010000:2.0:1769074933.244716:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.244725:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.244979:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 2/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.244994:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 2/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074933.245003:0:68501:0:(file.c:6914:ll_layout_lock_set()) ### file [0x240000404:0x4d:0x0](000000005a186648) being reconfigured ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.245049:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.245057:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074933.245072:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074933.245083:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074933.245086:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074933.245092:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074933.245104:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074933.245111:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074933.245133:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1769074933.245145:0:68501:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 2/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.245154:0:68501:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 2/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00010000:2.0:1769074933.245160:0:68501:0:(file.c:6914:ll_layout_lock_set()) ### file [0x240000404:0x4d:0x0](000000005a186648) being reconfigured ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:2.0:1769074933.245176:0:68501:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1769074933.245180:0:68501:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 3, move mirror from 0 to 0, have retried: 0, mirror count: 2 00020000:00200000:2.0:1769074933.245183:0:68501:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:2.0:1769074933.245185:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00020000:00200000:2.0:1769074933.245188:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[1] flags 0x10 00020000:00200000:2.0:1769074933.245192:0:68501:0:(lov_io.c:860:lov_io_iter_init()) component[2] flags 0x40 00000080:00200000:2.0:1769074933.245199:0:68501:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:2.0:1769074933.245211:0:68501:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 0 blocks 0 [cma]time 1769074933 1769074919 1769074907 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074933.245442:0:68501:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 70254592 blocks 2048 [cma]time: 1769074919 1769074919 0 by oinfo size 70254592 blocks 2048 [cma]time 1769074933 1769074919 1769074907 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 1/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x800020000020000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074933.245458:0:68501:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 0 blocks 0 [cma]time 0 0 0 ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 1/0,0 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000008:00010000:2.0:1769074933.245471:0:68501:0:(osc_object.c:209:osc_object_ast_clear()) ### update lock size 70254592 blocks 2048 [cma]time: 1769074933 1769074933 0 by oinfo size 70254592 blocks 2048 [cma]time 1769074933 1769074933 0 ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 1/0,0 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x800020000000000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00000080:00200000:2.0:1769074933.245525:0:68501:0:(file.c:6809:ll_layout_conf()) [0x240000404:0x4d:0x0]: layout version change: 4294967294 -> 8 00010000:00010000:2.0:1769074933.245531:0:68501:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 3/1,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:2.0:1769074933.245538:0:68501:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200008:2.0:1769074933.245577:0:68501:0:(file.c:424:ll_file_release()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8), flags 154002 00000002:00100000:2.0:1769074933.245624:0:68501:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0352a1b800 x1855007617063552/t4294968381(4294968381) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074987 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074933.245635:0:68501:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074933.245640:0:68501:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7100 x1855007617069056/t0(0) o35->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2192 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074933.245666:0:68501:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617069056:192.168.204.160@tcp:35:lfs.0 00000100:00100000:2.0:1769074933.246172:0:68501:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 11 seconds 00000100:00100000:2.0:1769074933.272463:0:68501:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68501:1855007617069056:192.168.204.160@tcp:35:lfs.0 00000002:00100000:2.0:1769074933.272500:0:68501:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0352a1b800 x1855007617063552/t4294968381(4294968381) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 1072/1160 e 0 to 0 dl 1769074987 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074933.287580:0:68502:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074933.287616:0:68502:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074933.287633:0:68502:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074933.287649:0:68502:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074933.287657:0:68502:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074933.287665:0:68502:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074933.287669:0:68502:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:0.0:1769074933.287676:0:68502:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364240fbf8 00010000:00010000:0.0:1769074933.287680:0:68502:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074933.287688:0:68502:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074933.287703:0:68502:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074933.287712:0:68502:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:0.0:1769074933.287719:0:68502:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:0.0:1769074933.287731:0:68502:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b0351f7e780, open_flags 18800, mode 0 opened 0 00000080:00200000:0.0:1769074933.287737:0:68502:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open 00000002:00010000:0.0:1769074933.291966:0:68502:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: open flags 0304001 00010000:00010000:0.0:1769074933.292056:0:68502:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879ca69 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68502 initiator: MDT0 00010000:00010000:0.0:1769074933.292068:0:68502:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879ca69 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68502 initiator: MDT0 00000020:00100000:0.0:1769074933.292081:0:68502:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074933.292094:0:68502:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a39c00 x1855007617069312/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074933.292104:0:68502:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879ca69 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68502 initiator: MDT0 00000100:00100000:0.0:1769074933.292127:0:68502:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68502:1855007617069312:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074933.292576:0:68502:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0345854a80 going to sleep for 50 seconds 00000080:00200008:0.0:1769074933.348549:0:68504:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074933.348561:0:68504:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074933.348585:0:68504:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074933.348604:0:68504:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074933.348621:0:68504:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074933.348630:0:68504:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074933.348639:0:68504:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074933.348642:0:68504:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:0.0:1769074933.348649:0:68504:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364241fcb8 00010000:00010000:0.0:1769074933.348654:0:68504:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074933.348663:0:68504:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074933.349106:0:68504:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074933.349117:0:68504:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074933.349124:0:68504:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074933.349133:0:68504:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074933.349140:0:68504:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074933.349144:0:68504:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074933.349146:0:68504:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:0.0:1769074933.349150:0:68504:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364241fbb0 00010000:00010000:0.0:1769074933.349154:0:68504:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074933.349162:0:68504:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074933.349170:0:68504:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:0.0:1769074933.349178:0:68504:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:0.0:1769074933.349195:0:68504:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffd0279b1dc 00000080:00200008:0.0:1769074933.349208:0:68504:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:1.0:1769074933.351340:0:68507:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074933.351370:0:68507:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074933.351386:0:68507:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.351409:0:68507:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.351417:0:68507:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074933.351425:0:68507:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074933.351429:0:68507:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074933.351435:0:68507:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364244fbf8 00010000:00010000:1.0:1769074933.351439:0:68507:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.351448:0:68507:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.351462:0:68507:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074933.351469:0:68507:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:1.0:1769074933.351475:0:68507:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:1.0:1769074933.351487:0:68507:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b0345579080, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1769074933.351494:0:68507:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open 00000002:00010000:1.0:1769074933.351510:0:68507:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: open flags 0304001 00010000:00010000:1.0:1769074933.351567:0:68507:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68507 initiator: MDT0 00010000:00010000:1.0:1769074933.351574:0:68507:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68507 initiator: MDT0 00000020:00100000:1.0:1769074933.351585:0:68507:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074933.351597:0:68507:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342cea300 x1855007617069696/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074933.351606:0:68507:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68507 initiator: MDT0 00000100:00100000:1.0:1769074933.351626:0:68507:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68507:1855007617069696:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074933.352057:0:68507:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a686c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074933.362601:0:68507:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68507:1855007617069696:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074933.362629:0:68507:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68507 initiator: MDT0 00010000:00010000:2.0:1769074933.362642:0:68507:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68507 initiator: MDT0 00010000:00010000:2.0:1769074933.362649:0:68507:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68507 initiator: MDT0 00010000:00010000:2.0:1769074933.362655:0:68507:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68507 initiator: MDT0 00010000:00010000:2.0:1769074933.362663:0:68507:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68507 initiator: MDT0 00010000:00010000:2.0:1769074933.362669:0:68507:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0348caec00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074933.362673:0:68507:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68507 initiator: MDT0 00010000:00010000:2.0:1769074933.362679:0:68507:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68507 initiator: MDT0 00010000:00010000:2.0:1769074933.362687:0:68507:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 5/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 68507 initiator: MDT0 00010000:00010000:2.0:1769074933.362694:0:68507:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074933.362701:0:68507:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68507 initiator: MDT0 00010000:00010000:2.0:1769074933.362707:0:68507:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348caec00/0xb7d6bf619879ca70 lrc: 0/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68507 initiator: MDT0 00000002:00100000:2.0:1769074933.362731:0:68507:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0342cea300 x1855007617069696/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/608 e 0 to 0 dl 1769074988 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074933.362746:0:68507:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x240000405:0xa:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074933.364222:0:68507:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074933.364270:0:68507:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074933.364287:0:68507:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.364302:0:68507:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074933.364310:0:68507:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074933.364317:0:68507:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074933.364320:0:68507:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074933.364326:0:68507:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364244fbf8 00010000:00010000:2.0:1769074933.364330:0:68507:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.364337:0:68507:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074933.364352:0:68507:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074933.364361:0:68507:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:2.0:1769074933.364368:0:68507:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 44 00000080:00200008:2.0:1769074933.364377:0:68507:0:(file.c:1001:ll_file_open()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000080:00200080:2.0:1769074933.364440:0:68507:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988) cmd=c1406916 arg=f97a10 00000100:00100000:2.0:1769074933.364509:0:68507:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68507:1855007617069824:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074933.364605:0:68507:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074933.375350:0:68507:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68507:1855007617069824:192.168.204.160@tcp:34:lfs.0 00000080:00200008:2.0:1769074933.375524:0:68507:0:(file.c:424:ll_file_release()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00010000:00010000:2.0:1769074933.375559:0:68507:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00000080:00200008:1.0:1769074933.420283:0:68511:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074933.420295:0:68511:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074933.420320:0:68511:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074933.420338:0:68511:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.420353:0:68511:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.420360:0:68511:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074933.420367:0:68511:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074933.420371:0:68511:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074933.420379:0:68511:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642457cb8 00010000:00010000:1.0:1769074933.420383:0:68511:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.420391:0:68511:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.420573:0:68511:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074933.420583:0:68511:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074933.420590:0:68511:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.420599:0:68511:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.420605:0:68511:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074933.420610:0:68511:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074933.420612:0:68511:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074933.420616:0:68511:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642457bb0 00010000:00010000:1.0:1769074933.420619:0:68511:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.420625:0:68511:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.420634:0:68511:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074933.420644:0:68511:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074933.420663:0:68511:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffea5f9c73c 00000080:00200008:1.0:1769074933.420678:0:68511:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:2.0:1769074933.422916:0:68515:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074933.422955:0:68515:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074933.422973:0:68515:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.422989:0:68515:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074933.422997:0:68515:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074933.423006:0:68515:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074933.423119:0:68515:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074933.423130:0:68515:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642467bf8 00010000:00010000:2.0:1769074933.423135:0:68515:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.423145:0:68515:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074933.423168:0:68515:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074933.423177:0:68515:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:2.0:1769074933.423186:0:68515:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:2.0:1769074933.423199:0:68515:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b03586ecf00, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074933.423206:0:68515:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open 00000002:00010000:2.0:1769074933.423223:0:68515:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074933.423306:0:68515:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68515 initiator: MDT0 00010000:00010000:2.0:1769074933.423315:0:68515:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68515 initiator: MDT0 00000020:00100000:2.0:1769074933.423329:0:68515:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074933.423342:0:68515:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7100 x1855007617070080/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074933.423351:0:68515:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68515 initiator: MDT0 00000100:00100000:2.0:1769074933.423373:0:68515:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68515:1855007617070080:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074933.423582:0:68515:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074933.438724:0:68515:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68515:1855007617070080:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074933.438749:0:68515:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68515 initiator: MDT0 00010000:00010000:2.0:1769074933.438763:0:68515:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68515 initiator: MDT0 00010000:00010000:2.0:1769074933.438774:0:68515:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68515 initiator: MDT0 00010000:00010000:2.0:1769074933.438782:0:68515:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68515 initiator: MDT0 00010000:00010000:2.0:1769074933.438790:0:68515:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68515 initiator: MDT0 00010000:00010000:2.0:1769074933.438813:0:68515:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365861c00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074933.438818:0:68515:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68515 initiator: MDT0 00010000:00010000:2.0:1769074933.438824:0:68515:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68515 initiator: MDT0 00010000:00010000:2.0:1769074933.438832:0:68515:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 5/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 68515 initiator: MDT0 00010000:00010000:2.0:1769074933.438840:0:68515:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074933.438847:0:68515:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68515 initiator: MDT0 00010000:00010000:2.0:1769074933.438853:0:68515:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879ca77 lrc: 0/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68515 initiator: MDT0 00000002:00100000:2.0:1769074933.438883:0:68515:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b03482c7100 x1855007617070080/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/608 e 0 to 0 dl 1769074988 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074933.438898:0:68515:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x240000405:0xa:0x0] suppgids 0 -1: rc -20 00000080:00200000:2.0:1769074933.439127:0:68515:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074933.439149:0:68515:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074933.439163:0:68515:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.439176:0:68515:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074933.439185:0:68515:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074933.439191:0:68515:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074933.439194:0:68515:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:2.0:1769074933.439199:0:68515:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642467bf8 00010000:00010000:2.0:1769074933.439203:0:68515:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074933.439211:0:68515:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074933.439224:0:68515:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074933.439292:0:68515:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:2.0:1769074933.439300:0:68515:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 44 00000080:00200008:2.0:1769074933.439309:0:68515:0:(file.c:1001:ll_file_open()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000080:00200080:2.0:1769074933.439357:0:68515:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988) cmd=c1406916 arg=185da10 00000100:00100000:2.0:1769074933.439429:0:68515:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68515:1855007617070208:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074933.439512:0:68515:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e3c0 going to sleep for 50 seconds 00000100:00100000:2.0:1769074933.445700:0:68515:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68515:1855007617070208:192.168.204.160@tcp:34:lfs.0 00000080:00200008:2.0:1769074933.445931:0:68515:0:(file.c:424:ll_file_release()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00010000:00010000:2.0:1769074933.445963:0:68515:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.493886:0:68520:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074933.493913:0:68520:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074933.493929:0:68520:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.493945:0:68520:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.493953:0:68520:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074933.493960:0:68520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074933.493964:0:68520:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074933.493970:0:68520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642447bf8 00010000:00010000:1.0:1769074933.493974:0:68520:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.493983:0:68520:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.493996:0:68520:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074933.494002:0:68520:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:1.0:1769074933.494008:0:68520:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:1.0:1769074933.494018:0:68520:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b0349371c80, open_flags 18800, mode 0 opened 0 00000080:00200000:1.0:1769074933.494024:0:68520:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open 00000002:00010000:1.0:1769074933.494039:0:68520:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: open flags 0304001 00010000:00010000:1.0:1769074933.494088:0:68520:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68520 initiator: MDT0 00010000:00010000:1.0:1769074933.494096:0:68520:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68520 initiator: MDT0 00000020:00100000:1.0:1769074933.494107:0:68520:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1769074933.494119:0:68520:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ce9180 x1855007617070464/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1769074933.494129:0:68520:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68520 initiator: MDT0 00000100:00100000:1.0:1769074933.494147:0:68520:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68520:1855007617070464:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074933.494339:0:68520:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a68540 going to sleep for 50 seconds 00000100:00100000:1.0:1769074933.501082:0:68520:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68520:1855007617070464:192.168.204.160@tcp:101:lfs.0 00010000:00010000:1.0:1769074933.501106:0:68520:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68520 initiator: MDT0 00010000:00010000:1.0:1769074933.501119:0:68520:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68520 initiator: MDT0 00010000:00010000:1.0:1769074933.501126:0:68520:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68520 initiator: MDT0 00010000:00010000:1.0:1769074933.501133:0:68520:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68520 initiator: MDT0 00010000:00010000:1.0:1769074933.501139:0:68520:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68520 initiator: MDT0 00010000:00010000:1.0:1769074933.501145:0:68520:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0348cae800 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:1.0:1769074933.501149:0:68520:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68520 initiator: MDT0 00010000:00010000:1.0:1769074933.501156:0:68520:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68520 initiator: MDT0 00010000:00010000:1.0:1769074933.501162:0:68520:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 5/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 68520 initiator: MDT0 00010000:00010000:1.0:1769074933.501172:0:68520:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1769074933.501179:0:68520:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68520 initiator: MDT0 00010000:00010000:1.0:1769074933.501185:0:68520:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0348cae800/0xb7d6bf619879ca7e lrc: 0/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68520 initiator: MDT0 00000002:00100000:1.0:1769074933.501210:0:68520:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0342ce9180 x1855007617070464/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/608 e 0 to 0 dl 1769074988 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1769074933.501225:0:68520:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x240000405:0xa:0x0] suppgids 0 -1: rc -20 00000080:00200000:1.0:1769074933.501604:0:68520:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074933.501629:0:68520:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074933.501642:0:68520:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.501653:0:68520:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.501660:0:68520:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074933.501667:0:68520:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074933.501670:0:68520:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074933.501675:0:68520:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb43642447bf8 00010000:00010000:1.0:1769074933.501678:0:68520:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.501686:0:68520:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.501700:0:68520:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074933.501709:0:68520:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:1.0:1769074933.501717:0:68520:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 44 00000080:00200008:1.0:1769074933.501726:0:68520:0:(file.c:1001:ll_file_open()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000080:00200080:1.0:1769074933.501774:0:68520:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988) cmd=c1406916 arg=ea8a10 00000100:00100000:1.0:1769074933.502009:0:68520:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68520:1855007617070592:192.168.204.160@tcp:34:lfs.0 00000100:00100000:1.0:1769074933.502128:0:68520:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a68f00 going to sleep for 50 seconds 00000080:00200008:1.0:1769074933.534126:0:68525:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074933.534137:0:68525:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074933.534180:0:68525:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074933.534198:0:68525:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.534214:0:68525:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.534222:0:68525:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074933.534229:0:68525:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074933.534314:0:68525:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074933.534322:0:68525:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364240fcb8 00010000:00010000:1.0:1769074933.534326:0:68525:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.534336:0:68525:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.534642:0:68525:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074933.534655:0:68525:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074933.534661:0:68525:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.534671:0:68525:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.534677:0:68525:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074933.534681:0:68525:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074933.534684:0:68525:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074933.534687:0:68525:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364240fbb0 00010000:00010000:1.0:1769074933.534691:0:68525:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074933.534698:0:68525:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074933.534706:0:68525:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074933.534714:0:68525:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074933.534737:0:68525:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffec1f92a1c 00000080:00200008:1.0:1769074933.534748:0:68525:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00010000:00010000:3.0:1769074933.556552:0:68529:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074933.589179:0:68531:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:3.0:1769074933.589261:0:68531:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074933.589281:0:68531:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074933.589297:0:68531:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074933.589304:0:68531:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074933.589312:0:68531:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:3.0:1769074933.589316:0:68531:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:3.0:1769074933.589322:0:68531:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb436424b7bf8 00010000:00010000:3.0:1769074933.589326:0:68531:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074933.589335:0:68531:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074933.589350:0:68531:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:3.0:1769074933.589358:0:68531:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=27 00000080:00200000:3.0:1769074933.589364:0:68531:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:3.0:1769074933.589376:0:68531:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), file ffff9b0343b79c80, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1769074933.589382:0:68531:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=open 00000002:00010000:3.0:1769074933.589408:0:68531:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: open flags 0304001 00010000:00010000:3.0:1769074933.590724:0:68531:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68531 initiator: MDT0 00010000:00010000:3.0:1769074933.590747:0:68531:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68531 initiator: MDT0 00000020:00100000:3.0:1769074933.590847:0:68531:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1769074933.590862:0:68531:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0352a18e00 x1855007617071232/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/133424 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1769074933.590873:0:68531:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68531 initiator: MDT0 00000100:00100000:3.0:1769074933.590901:0:68531:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68531:1855007617071232:192.168.204.160@tcp:101:lfs.0 00000100:00100000:3.0:1769074933.591149:0:68531:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1ab40 going to sleep for 50 seconds 00000100:00100000:3.0:1769074933.600182:0:68531:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68531:1855007617071232:192.168.204.160@tcp:101:lfs.0 00010000:00010000:3.0:1769074933.600227:0:68531:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68531 initiator: MDT0 00010000:00010000:3.0:1769074933.600279:0:68531:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68531 initiator: MDT0 00010000:00010000:3.0:1769074933.600288:0:68531:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68531 initiator: MDT0 00010000:00010000:3.0:1769074933.600295:0:68531:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 68531 initiator: MDT0 00010000:00010000:3.0:1769074933.600306:0:68531:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 68531 initiator: MDT0 00010000:00010000:3.0:1769074933.600315:0:68531:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919c200 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:3.0:1769074933.600320:0:68531:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68531 initiator: MDT0 00010000:00010000:3.0:1769074933.600336:0:68531:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 4/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 68531 initiator: MDT0 00010000:00010000:3.0:1769074933.600343:0:68531:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 5/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 68531 initiator: MDT0 00010000:00010000:3.0:1769074933.600651:0:68531:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074933.600660:0:68531:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 3/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68531 initiator: MDT0 00010000:00010000:3.0:1769074933.600667:0:68531:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919c200/0xb7d6bf619879ca8c lrc: 0/0,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 68531 initiator: MDT0 00000002:00100000:3.0:1769074933.600693:0:68531:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0352a18e00 x1855007617071232/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2232/608 e 0 to 0 dl 1769074988 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1769074933.600707:0:68531:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x240000405:0xa:0x0] suppgids 0 -1: rc -20 00000080:00200000:0.0:1769074933.601003:0:68531:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074933.601028:0:68531:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074933.601041:0:68531:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074933.601053:0:68531:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074933.601060:0:68531:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074933.601065:0:68531:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074933.601069:0:68531:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:0.0:1769074933.601073:0:68531:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb436424b7bf8 00010000:00010000:0.0:1769074933.601076:0:68531:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074933.601083:0:68531:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074933.601095:0:68531:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074933.601102:0:68531:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=267 00000080:00200000:0.0:1769074933.601108:0:68531:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 44 00000080:00200008:0.0:1769074933.601116:0:68531:0:(file.c:1001:ll_file_open()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00000080:00200080:0.0:1769074933.601151:0:68531:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988) cmd=c0086915 arg=1c083d0 00000100:00100000:0.0:1769074933.601210:0:68531:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68531:1855007617071360:192.168.204.160@tcp:34:lfs.0 00000100:00100000:0.0:1769074933.601407:0:68531:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0345854900 going to sleep for 50 seconds 00000100:00100000:0.0:1769074933.605620:0:68531:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68531:1855007617071360:192.168.204.160@tcp:34:lfs.0 00000080:00200008:0.0:1769074933.605797:0:68531:0:(file.c:424:ll_file_release()) START file d204d.sanity-flr:[0x200000405:0x20c:0x0](ffff9b0355d41988), flags 304000 00010000:00010000:0.0:1769074933.605830:0:68531:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074934.231359:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074935.253842:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074936.276453:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074936.405007:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617071488:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074936.405107:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617071744:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074936.405133:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617072000:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074936.405160:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074936.405166:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617071616:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074936.405188:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617071872:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.418805:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617071616:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.418826:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617071872:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.418836:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617071488:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.418845:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617071744:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074936.418855:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617072000:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769074937.299940:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074938.334623:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074939.348467:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1769074939.386642:0:68956:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074939.386674:0:68956:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074939.386694:0:68956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074939.386711:0:68956:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074939.386718:0:68956:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074939.386726:0:68956:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074939.386730:0:68956:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:0.0:1769074939.386737:0:68956:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364155bb58 00010000:00010000:0.0:1769074939.386741:0:68956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074939.386749:0:68956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074939.386764:0:68956:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074939.386772:0:68956:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=24 00000080:00200000:0.0:1769074939.386778:0:68956:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:0.0:1769074939.386788:0:68956:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), flags=8 00000080:00200000:0.0:1769074939.386794:0:68956:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988), intent=getattr 00000002:00010000:0.0:1769074939.386813:0:68956:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204d.sanity-flr,[0x0:0x0:0x0]) in obj [0x240000405:0xa:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074939.386872:0:68956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 68956 initiator: MDT0 00010000:00010000:0.0:1769074939.386881:0:68956:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68956 initiator: MDT0 00010000:00010000:0.0:1769074939.386891:0:68956:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 68956 initiator: MDT0 00000100:00100000:0.0:1769074939.386916:0:68956:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68956:1855007617072128:192.168.204.160@tcp:101:rm.0 00000100:00100000:0.0:1769074939.387034:0:68956:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b03569ca180 going to sleep for 50 seconds 00000100:00100000:0.0:1769074939.393894:0:68956:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9b0342ad2800: committing for last_committed 4294968390 gen 1 00000100:00100000:0.0:1769074939.393914:0:68956:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68956:1855007617072128:192.168.204.160@tcp:101:rm.0 00010000:00010000:0.0:1769074939.393937:0:68956:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xa:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074939.393966:0:68956:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 4/1,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074939.393974:0:68956:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 4/1,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074939.393985:0:68956:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 4/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074939.393995:0:68956:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 4/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074939.394007:0:68956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 5/2,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074939.394015:0:68956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 5/2,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074939.394022:0:68956:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 4/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074939.394034:0:68956:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b0365a38380 x1855007617072128/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/1136 e 0 to 0 dl 1769074994 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074939.394045:0:68956:0:(mdc_locks.c:937:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 552 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 0 00000002:00010000:0.0:1769074939.394058:0:68956:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074939.394067:0:68956:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 4 res 9663677444/77 (0 0) 00000080:00200000:0.0:1769074939.394074:0:68956:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000405:0xa:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1769074939.394209:0:68956:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x240000404:0x4d:0x0], updating i_size 70254592 00000080:00200000:0.0:1769074939.394213:0:68956:0:(namei.c:151:ll_iget()) got inode: [0x240000404:0x4d:0x0](ffff9b0355ce21c8): rc = 0 00000080:00200000:0.0:1769074939.394386:0:68956:0:(file.c:6809:ll_layout_conf()) [0x240000404:0x4d:0x0]: layout version change: 8 -> 8 00000080:00010000:0.0:1769074939.394393:0:68956:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca93 00000080:00010000:0.0:1769074939.394405:0:68956:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000080:00010000:0.0:1769074939.394409:0:68956:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca93 from it ffffb4364155bb20 00010000:00010000:0.0:1769074939.394412:0:68956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 3 00010000:00010000:0.0:1769074939.394424:0:68956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 3 00000080:00200008:1.0:1769074939.397223:0:68956:0:(file.c:6252:ll_getattr_dentry()) START file f204d.sanity-flr:[0x240000404:0x4d:0x0](ffff9b0355ce21c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1769074939.397266:0:68956:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8),name=f204d.sanity-flr 00000002:00010000:1.0:1769074939.397312:0:68956:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000404:0x4d:0x0]) in obj [0x240000405:0xa:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074939.397331:0:68956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074939.397345:0:68956:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 2/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074939.397353:0:68956:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x240000405:0xa:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074939.397360:0:68956:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00000080:00010000:1.0:1769074939.397364:0:68956:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x240000404:0x4d:0x0](ffff9b0355ce21c8) for lock 0xb7d6bf619879ca93 00000080:00010000:1.0:1769074939.397371:0:68956:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879ca93 from it ffffb4364155bcb8 00010000:00010000:1.0:1769074939.397375:0:68956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 3/1,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074939.397382:0:68956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074939.397445:0:68956:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074939.397454:0:68956:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074939.397463:0:68956:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074939.397471:0:68956:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074939.397478:0:68956:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074939.397482:0:68956:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074939.397485:0:68956:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074939.397489:0:68956:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364155bc10 00010000:00010000:1.0:1769074939.397492:0:68956:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074939.397499:0:68956:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074939.397509:0:68956:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074939.397515:0:68956:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204d.sanity-flr, flags=24 00000080:00200000:1.0:1769074939.397520:0:68956:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 1 00000080:00200000:1.0:1769074939.397526:0:68956:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204d.sanity-flr, flags=0 00000080:00200000:1.0:1769074939.397529:0:68956:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20c:0x0](ffff9b0355d41988), inode mode 41ed mask 3 00000080:00200000:1.0:1769074939.397534:0:68956:0:(namei.c:2354:ll_unlink()) VFS Op:name=f204d.sanity-flr, dir=[0x200000405:0x20c:0x0](ffff9b0355d41988) 00010000:00010000:1.0:1769074939.397562:0:68956:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 2/0,0 mode: CR/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074939.397576:0:68956:0:(vvp_object.c:114:vvp_conf_set()) [0x240000404:0x4d:0x0]: losing layout lock 00010000:00010000:1.0:1769074939.397590:0:68956:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 3 00000080:00200000:1.0:1769074939.397610:0:68956:0:(vvp_object.c:114:vvp_conf_set()) [0x240000404:0x4d:0x0]: losing layout lock 00010000:00010000:1.0:1769074939.397619:0:68956:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005ac00/0xb7d6bf619879c9cf lrc: 2/0,0 mode: PR/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x20/0x0 rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55e1f expref: -99 pid: 68465 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074939.398083:0:68956:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 1/0,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074939.398094:0:68956:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 1/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074939.398101:0:68956:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005ac00/0xb7d6bf619879c9cf lrc: 1/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55e1f expref: -99 pid: 68465 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074939.398106:0:68956:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:1.0:1769074939.398109:0:68956:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879ca62 lrc: 0/0,0 mode: --/CR res: [0x240000404:0x4d:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd55fd1 expref: -99 pid: 68501 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074939.398134:0:68956:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ec00/0xb7d6bf619879ca93 lrc: 0/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x7fe2323adfd560e9 expref: -99 pid: 68956 timeout: 0 lvb_type: 3 00010000:00010000:1.0:1769074939.398143:0:68956:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005ac00/0xb7d6bf619879c9cf lrc: 0/0,0 mode: --/PR res: [0x240000404:0x4d:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55e1f expref: -99 pid: 68465 timeout: 0 lvb_type: 0 00000020:00100000:1.0:1769074939.398166:0:68956:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1769074939.398178:0:68956:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0342ce8a80 x1855007617072384/t0(0) o36->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 512/2192 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074939.398201:0:68956:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68956:1855007617072384:192.168.204.160@tcp:36:rm.0 00000100:00100000:1.0:1769074939.398441:0:68956:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0348fc1180 going to sleep for 50 seconds 00000100:00100000:1.0:1769074939.408637:0:68956:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job rm:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:68956:1855007617072384:192.168.204.160@tcp:36:rm.0 00000080:00200000:1.0:1769074939.408706:0:68956:0:(vvp_io.c:1854:vvp_io_init()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1769074939.408719:0:68956:0:(lov_io.c:458:lov_io_mirror_init()) [0x240000404:0x4d:0x0]: flr state: 1, move mirror from 1 to 1, have retried: 0, mirror count: 2 00020000:00200000:1.0:1769074939.408722:0:68956:0:(lov_io.c:484:lov_io_mirror_init()) use delayed RPC state for this IO 00020000:00200000:1.0:1769074939.408728:0:68956:0:(lov_io.c:860:lov_io_iter_init()) component[3] flags 0x10 00020000:00200000:1.0:1769074939.408769:0:68956:0:(lov_io.c:860:lov_io_iter_init()) component[4] flags 0x10 00000080:00200000:1.0:1769074939.408806:0:68956:0:(vvp_io.c:289:vvp_io_fini()) [0x240000404:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1769074939.408814:0:68956:0:(llite_lib.c:1953:ll_clear_inode()) VFS Op:inode=[0x240000404:0x4d:0x0](ffff9b0355ce21c8) 00010000:00010000:3.0:1769074940.371437:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1769074940.758384:0:69097:0:(debug.c:665:libcfs_debug_mark_buffer()) DEBUG MARKER: == sanity-flr test 204e: FLR write/stale/resync sel test with repeated comp ========================================================== 04:42:19 (1769074939) 00010000:00010000:0.0:1769074941.395462:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074941.525880:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2408:1855007617072512:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074941.525900:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617072640:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074941.525987:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617072896:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074941.525987:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617072768:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074941.526021:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617073024:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074941.536513:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2408:1855007617072512:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074941.536540:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617072768:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074941.536570:0:2408:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9b0342ad2800: committing for last_committed 17179874650 gen 2 00000100:00100000:3.0:1769074941.537488:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617073024:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074941.538089:0:2409:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9b0342ad2800: committing for last_committed 8589944144 gen 1 00000100:00100000:2.0:1769074941.538104:0:2409:0:(client.c:3130:ptlrpc_free_committed()) @@@ keeping (FL_REPLAY) req@ffff9b0365a3a680 x1855007617062144/t8589944144(8589944144) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074987 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769074941.538114:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617072640:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074941.538139:0:2409:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9b0342ad2800: committing for last_committed 21474841319 gen 1 00000100:00100000:2.0:1769074941.538153:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617072896:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769074942.421286:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074943.108525:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074943.108573:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074943.108675:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625289216 00000100:00100000:0.0:1769074943.108686:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625289216 00000100:00100000:0.0:1769074943.108740:0:39330:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625289344 00000100:00100000:0.0:1769074943.108744:0:39330:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625289344 00000100:00100000:0.0:1769074943.108760:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8966 00000100:00100000:0.0:1769074943.108768:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:27843:x1855007625289216:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074943.108785:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074943.108867:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074943.108874:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9500 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:27843:x1855007625289216:12345-192.168.204.160@tcp:104: Request processed in 106us (359us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074943.108881:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8966 00000100:00100000:0.0:1769074943.108898:0:39330:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8967 00000100:00100000:0.0:1769074943.108904:0:39330:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ce9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28519:x1855007625289344:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074943.108911:0:39330:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769074943.109043:0:39330:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:0.0:1769074943.109050:0:39330:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ce9880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:28519:x1855007625289344:12345-192.168.204.160@tcp:104: Request processed in 146us (475us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074943.109066:0:39330:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8967 00010000:00010000:3.0:1769074943.112517:0:46917:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1769074943.112531:0:46917:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.112543:0:46917:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860a00 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:3.0:1769074943.112550:0:46917:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x428400010000 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.112621:0:46917:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 2/0,0 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.112628:0:46917:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074943.112659:0:46917:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 1/0,0 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.112667:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879ca4d lrc: 0/0,0 mode: --/PW res: [0x280000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 10485760->18446744073709551615) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55fae expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.112691:0:46917:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074943.112695:0:46917:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 2/0,0 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.112700:0:46917:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365861000 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:3.0:1769074943.112704:0:46917:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 3/0,0 mode: PW/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x428400030000 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.112724:0:46917:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 2/0,0 mode: --/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.112730:0:46917:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074943.112735:0:46917:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 1/0,0 mode: --/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.112742:0:46917:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0365861000/0xb7d6bf619879ca54 lrc: 0/0,0 mode: --/PW res: [0x280000400:0x36:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->67108863) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55fbc expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.112753:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074943.112754:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074943.112756:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074943.112775:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617073280:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1769074943.112899:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074943.113305:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617073152:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074943.113746:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:1.0:1769074943.119698:0:39329:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625289600 00000100:00100000:1.0:1769074943.119715:0:39329:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625289600 00000100:00100000:1.0:1769074943.119786:0:39329:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8968 00000100:00100000:1.0:1769074943.119796:0:39329:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0342ceb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:36516:x1855007625289600:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074943.119812:0:39329:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074943.119898:0:39329:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074943.119904:0:39329:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0342ceb480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:36516:x1855007625289600:12345-192.168.204.160@tcp:104: Request processed in 110us (6161us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074943.119911:0:39329:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8968 00010000:00010000:3.0:1769074943.128365:0:60221:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074943.128382:0:60221:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x420000030000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.128394:0:60221:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365861e00 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:3.0:1769074943.128402:0:60221:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x428400030000 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.128465:0:60221:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 2/0,0 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.128478:0:60221:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:2.0:1769074943.128565:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617073408:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1769074943.129523:0:60221:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.129537:0:60221:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365861e00/0xb7d6bf619879ca5b lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0x38:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->134217727) gid 0 flags: 0x4c29400030020 nid: local remote: 0x7fe2323adfd55fc3 expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.129562:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074943.129565:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074943.129567:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074943.133462:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617073280:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074943.134074:0:2399:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:2.0:1769074943.135022:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617073152:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074943.135048:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617073408:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1769074943.135324:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625289856 00000100:00100000:1.0:1769074943.135332:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625289856 00000100:00100000:1.0:1769074943.135431:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8969 00000100:00100000:1.0:1769074943.135442:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0365a38380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:27843:x1855007625289856:12345-192.168.204.160@tcp:104: 00010000:00010000:1.0:1769074943.135457:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769074943.135533:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1769074943.135540:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0365a38380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:27843:x1855007625289856:12345-192.168.204.160@tcp:104: Request processed in 101us (1470us total) trans 0 rc 0/0 00000100:00100000:1.0:1769074943.135548:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8969 00010000:00010000:3.0:1769074943.135610:0:39340:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074943.135618:0:39340:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 2/0,0 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x420000010000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.135625:0:39340:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0365860200 already unused, calling callback (ffffffffc11aa1b0) 00010000:00010000:3.0:1769074943.135631:0:39340:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 3/0,0 mode: PW/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x428400010000 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.135676:0:39340:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 2/0,0 mode: --/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.135684:0:39340:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074943.135715:0:39340:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 1/0,0 mode: --/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.135723:0:39340:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0365860200/0xb7d6bf619879ca38 lrc: 0/0,0 mode: --/PW res: [0x2c0000400:0x37:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->10485759) gid 0 flags: 0x4c29400010020 nid: local remote: 0x7fe2323adfd55f7d expref: -99 pid: 68501 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769074943.135747:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074943.135749:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074943.135751:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769074943.135758:0:2409:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1769074943.135769:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617073536:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1769074943.145987:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617073536:192.168.204.160@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1769074943.446184:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074944.469389:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074945.491930:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074946.516907:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074946.646698:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617073792:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074946.646794:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074048:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074946.646830:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:0.0:1769074946.646836:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617073664:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074946.646861:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617073920:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074946.646881:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074176:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074946.650108:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617073792:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074946.653114:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074048:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074946.653674:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617073664:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074946.653687:0:2406:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff9b0342ad2800: committing for last_committed 4294968391 gen 1 00000100:00100000:0.0:1769074946.653698:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617073920:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074946.653919:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074176:192.168.204.160@tcp:400:kworker.0 00010000:00010000:2.0:1769074947.541171:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074948.563821:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074949.587686:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074950.611915:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074951.635883:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074951.764515:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617074304:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074951.764610:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074560:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074951.764639:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074816:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074951.764669:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1769074951.764676:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074432:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074951.764699:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074688:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074951.768511:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074432:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074951.768532:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074560:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074951.770000:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074688:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074951.770016:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617074304:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074951.770026:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617074816:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769074952.659668:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074953.684138:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074954.707475:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074955.731520:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074956.755519:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074956.883644:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617074944:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074956.883755:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617075200:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074956.883782:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617075456:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074956.883810:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074956.883817:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617075072:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074956.883839:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617075328:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074956.887219:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617075072:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074956.887264:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617075328:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074956.887288:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617075200:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074956.887298:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617075456:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074956.888169:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617074944:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769074957.779573:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074958.803445:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1769074959.059766:0:69555:0:(llite_lib.c:2766:ll_statfs()) VFS Op:sb=lustre (ffff9b0342ad2800) / proj=0 00000080:00000004:2.0:1769074959.059777:0:69555:0:(obd_class.h:1083:obd_statfs()) lustre-clilmv-ffff9b0342ad2800: age 1901, max_age 1952 00800000:00000004:2.0:1769074959.059791:0:69555:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff9b0342ad2800: age 1901, max_age 1952 00000100:00100000:2.0:1769074959.059837:0:69555:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job stat:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69555:1855007617075584:192.168.204.160@tcp:41:stat.0 00000100:00100000:2.0:1769074959.059989:0:69555:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e900 going to sleep for 50 seconds 00000100:00100000:2.0:1769074959.068752:0:69555:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job stat:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69555:1855007617075584:192.168.204.160@tcp:41:stat.0 00800000:00000004:2.0:1769074959.068780:0:69555:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0000-mdc-ffff9b0342ad2800: update ffff9b03512de6d0 cache blocks 1798805/1916558 objects 523390/524075 00000080:00000004:2.0:1769074959.068787:0:69555:0:(obd_class.h:1109:obd_statfs()) lustre-clilmv-ffff9b0342ad2800: update ffff9b03512db520 cache blocks 1798805/1916558 objects 523390/524075 00000080:00000004:2.0:1769074959.068793:0:69555:0:(llite_lib.c:2629:ll_statfs_internal()) MDC blocks 1798805/1916558 objects 523390/524075 00000080:00200000:1.0:1769074959.114666:0:69559:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074959.114699:0:69559:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074959.114716:0:69559:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074959.114732:0:69559:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074959.114740:0:69559:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074959.114747:0:69559:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074959.114751:0:69559:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:1.0:1769074959.114759:0:69559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364233fb58 00010000:00010000:1.0:1769074959.114762:0:69559:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074959.114771:0:69559:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074959.114785:0:69559:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:1.0:1769074959.114796:0:69559:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d204e.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), flags=13 00000080:00200000:1.0:1769074959.114800:0:69559:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d204e.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=getattr 00000002:00010000:1.0:1769074959.114812:0:69559:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074959.114855:0:69559:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69559 initiator: MDT0 00010000:00010000:1.0:1769074959.114862:0:69559:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69559 initiator: MDT0 00010000:00010000:1.0:1769074959.114871:0:69559:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69559 initiator: MDT0 00000100:00100000:1.0:1769074959.114893:0:69559:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69559:1855007617075712:192.168.204.160@tcp:101:lfs.0 00000100:00100000:1.0:1769074959.115049:0:69559:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b035da83c00 going to sleep for 50 seconds 00000100:00100000:2.0:1769074959.139124:0:69559:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69559:1855007617075712:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074959.139415:0:69559:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69559 initiator: MDT0 00010000:00010000:2.0:1769074959.139429:0:69559:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69559 initiator: MDT0 00010000:00010000:2.0:1769074959.139436:0:69559:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69559 initiator: MDT0 00010000:00010000:2.0:1769074959.139443:0:69559:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69559 initiator: MDT0 00010000:00010000:2.0:1769074959.139450:0:69559:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69559 initiator: MDT0 00010000:00010000:2.0:1769074959.139456:0:69559:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b0348cae000 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074959.139460:0:69559:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 69559 initiator: MDT0 00010000:00010000:2.0:1769074959.139467:0:69559:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 69559 initiator: MDT0 00010000:00010000:2.0:1769074959.139474:0:69559:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69559 initiator: MDT0 00010000:00010000:2.0:1769074959.139480:0:69559:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074959.139487:0:69559:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69559 initiator: MDT0 00010000:00010000:2.0:1769074959.139493:0:69559:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0348cae000/0xb7d6bf619879ca9a lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69559 initiator: MDT0 00000002:00100000:2.0:1769074959.139510:0:69559:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9b0342cead80 x1855007617075712/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/584 e 0 to 0 dl 1769075014 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074959.139525:0:69559:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1769074959.139649:0:69559:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.139663:0:69559:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00010000:3.0:1769074959.139672:0:69559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364233f940 00010000:00010000:3.0:1769074959.139676:0:69559:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.139683:0:69559:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.139864:0:69559:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:3.0:1769074959.139879:0:69559:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074959.139886:0:69559:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.139894:0:69559:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.139901:0:69559:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074959.139906:0:69559:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:3.0:1769074959.139909:0:69559:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:3.0:1769074959.139913:0:69559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364233fbe8 00010000:00010000:3.0:1769074959.139916:0:69559:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.139922:0:69559:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.139929:0:69559:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:3.0:1769074959.139936:0:69559:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=16392 00000080:00200008:3.0:1769074959.139957:0:69559:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1769074959.139962:0:69559:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:3.0:1769074959.139967:0:69559:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074959.139984:0:69559:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.139991:0:69559:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.139998:0:69559:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074959.140001:0:69559:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:3.0:1769074959.140004:0:69559:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:3.0:1769074959.140007:0:69559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364233fcb8 00010000:00010000:3.0:1769074959.140010:0:69559:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.140016:0:69559:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200008:3.0:1769074959.142727:0:69559:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1769074959.142738:0:69559:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:3.0:1769074959.142759:0:69559:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074959.142775:0:69559:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.142790:0:69559:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.142799:0:69559:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074959.142807:0:69559:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:3.0:1769074959.142811:0:69559:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:3.0:1769074959.142818:0:69559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364233fcb8 00010000:00010000:3.0:1769074959.142821:0:69559:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.142829:0:69559:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.142989:0:69559:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:3.0:1769074959.142997:0:69559:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074959.143003:0:69559:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.143011:0:69559:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.143017:0:69559:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074959.143021:0:69559:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:3.0:1769074959.143023:0:69559:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879c974 00000080:00010000:3.0:1769074959.143026:0:69559:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879c974 from it ffffb4364233fbb0 00010000:00010000:3.0:1769074959.143029:0:69559:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.143036:0:69559:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.143043:0:69559:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:3.0:1769074959.143051:0:69559:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:3.0:1769074959.143069:0:69559:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=c03066f0 arg=7fff24857340 00000080:00200000:3.0:1769074959.143082:0:69559:0:(dir.c:483:ll_dir_setdirstripe()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) name=d204e.sanity-flr stripe_offset=0 stripe_count=2, hash_type=3 00010000:00010000:3.0:1769074959.143104:0:69559:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.143160:0:69559:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.143172:0:69559:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074959.143175:0:69559:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ba00/0xb7d6bf619879c974 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd55d0e expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00000020:00100000:3.0:1769074959.143726:0:69559:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1769074959.143739:0:69559:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0352a1bb80 x1855007617075968/t0(0) o36->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 568/2080 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074959.143765:0:69559:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1bb80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69559:1855007617075968:192.168.204.160@tcp:36:lfs.0 00000100:00100000:3.0:1769074959.143917:0:69559:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a180 going to sleep for 50 seconds 00000100:00100000:3.0:1769074959.159013:0:69559:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1bb80 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69559:1855007617075968:192.168.204.160@tcp:36:lfs.0 00000080:00200000:3.0:1769074959.159072:0:69559:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8) 00000080:00200000:3.0:1769074959.159079:0:69559:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20d:0x0], updating i_size 4096 00000080:00200008:3.0:1769074959.159115:0:69559:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:2.0:1769074959.178843:0:69562:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074959.178875:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074959.178941:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 69562 initiator: MDT0 00010000:00010000:2.0:1769074959.178951:0:69562:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00010000:00010000:2.0:1769074959.178962:0:69562:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00000100:00100000:2.0:1769074959.183495:0:69562:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076096:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074959.183655:0:69562:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ed80 going to sleep for 50 seconds 00000100:00100000:2.0:1769074959.192274:0:69562:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076096:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074959.192301:0:69562:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.192313:0:69562:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.192326:0:69562:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.192338:0:69562:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.192351:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.192359:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.192367:0:69562:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074959.192377:0:69562:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b03482c7480 x1855007617076096/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 576/688 e 0 to 0 dl 1769075014 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074959.192393:0:69562:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.192406:0:69562:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1769074959.192413:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074959.192441:0:69562:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:2.0:1769074959.192448:0:69562:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:2.0:1769074959.192453:0:69562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074959.192457:0:69562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423e7b58 00010000:00010000:2.0:1769074959.192460:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.192468:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074959.192493:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074959.192513:0:69562:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=d204e.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), flags=29 00000080:00200000:2.0:1769074959.192518:0:69562:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=d204e.sanity-flr, dir=[0x200000007:0x1:0x0](ffff9b0352593a88), intent=0 00000002:00010000:2.0:1769074959.192540:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: d204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:2.0:1769074959.192579:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69562 initiator: MDT0 00010000:00010000:2.0:1769074959.192585:0:69562:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00010000:00010000:2.0:1769074959.192593:0:69562:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00000100:00100000:2.0:1769074959.192611:0:69562:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076224:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074959.192680:0:69562:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ed80 going to sleep for 50 seconds 00000100:00100000:2.0:1769074959.203074:0:69562:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076224:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074959.203101:0:69562:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.203125:0:69562:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.203134:0:69562:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.203147:0:69562:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.203163:0:69562:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.203175:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.203182:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.203190:0:69562:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074959.203201:0:69562:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=10 disposition=b, status=0 req@ffff9b03482c7480 x1855007617076224/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/672 e 0 to 0 dl 1769075014 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074959.203216:0:69562:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.203227:0:69562:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/525 (0 0) 00000080:00200000:2.0:1769074959.203277:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000002:00010000:2.0:1769074959.203345:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000bd0:0xb:0x0]) in obj [0x200000bd0:0xb:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074959.203396:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 69562 initiator: MDT0 00010000:00010000:2.0:1769074959.203404:0:69562:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00010000:00010000:2.0:1769074959.203414:0:69562:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00000100:00100000:2.0:1769074959.203438:0:69562:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076352:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074959.203513:0:69562:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ed80 going to sleep for 50 seconds 00000100:00100000:2.0:1769074959.208055:0:69562:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076352:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074959.208080:0:69562:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.208093:0:69562:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 4/1,0 mode: --/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.208104:0:69562:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 4/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.208114:0:69562:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 4/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.208125:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 5/2,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.208133:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 5/2,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.208140:0:69562:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 4/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769074959.208151:0:69562:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b03482c6680 x1855007617076352/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 576/640 e 0 to 0 dl 1769075014 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1769074959.208166:0:69562:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 3/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.208179:0:69562:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589937616/11 (0 0) 00010000:00010000:2.0:1769074959.208187:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 3/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074959.208196:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 2/0,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000002:00010000:2.0:1769074959.208225:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000405:0xb:0x0]) in obj [0x240000405:0xb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074959.211902:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xb:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.211920:0:69562:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.211933:0:69562:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 3/1,0 mode: --/CR res: [0x240000405:0xb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00000100:00100000:0.0:1769074959.211957:0:69562:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076480:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074959.212060:0:69562:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61d80 going to sleep for 50 seconds 00000100:00100000:0.0:1769074959.227517:0:69562:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076480:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074959.227544:0:69562:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 4/1,0 mode: --/CR res: [0x240000405:0xb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.227556:0:69562:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 4/1,0 mode: --/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.227565:0:69562:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 4/1,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.227575:0:69562:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 4/1,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.227587:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 5/2,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.227594:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 5/2,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.227602:0:69562:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 4/1,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074959.227613:0:69562:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b0365a3a300 x1855007617076480/t0(0) o101->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 576/640 e 0 to 0 dl 1769075014 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074959.227629:0:69562:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 3/1,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.227642:0:69562:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b03530ca800 type 13 mode 4 res 9663677445/11 (0 0) 00010000:00010000:0.0:1769074959.227650:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 3/1,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.227658:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 2/0,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.227688:0:69562:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20d:0x0], updating i_size 8192 00000080:00200000:0.0:1769074959.227692:0:69562:0:(namei.c:151:ll_iget()) got inode: [0x200000405:0x20d:0x0](ffff9b0355ce63c8): rc = 0 00000080:00010000:0.0:1769074959.227701:0:69562:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20d:0x0](ffff9b0355ce63c8) for lock 0xb7d6bf619879caa8 00000080:00010000:0.0:1769074959.227711:0:69562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20d:0x0](ffff9b0355ce63c8) 00000080:00010000:0.0:1769074959.227714:0:69562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa8 from it ffffb436423e7a28 00010000:00010000:0.0:1769074959.227718:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.227726:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365861c00/0xb7d6bf619879caa8 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5618a expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.227750:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), inode mode 41ed mask 1 00000080:00200000:0.0:1769074959.227769:0:69562:0:(namei.c:1277:ll_lookup_nd()) VFS Op:name=f204e.sanity-flr, dir=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), flags=13 00000080:00200000:0.0:1769074959.227773:0:69562:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), intent=getattr 00000002:00010000:0.0:1769074959.227794:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0xb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074959.227830:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.227837:0:69562:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.227846:0:69562:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00000100:00100000:0.0:1769074959.227864:0:69562:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076608:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074959.227980:0:69562:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61d80 going to sleep for 50 seconds 00000100:00100000:0.0:1769074959.235991:0:69562:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076608:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074959.236020:0:69562:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236033:0:69562:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236039:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236045:0:69562:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236052:0:69562:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236057:0:69562:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919e200 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074959.236061:0:69562:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236068:0:69562:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236079:0:69562:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236085:0:69562:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074959.236091:0:69562:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236096:0:69562:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cabd lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69562 initiator: MDT0 00000002:00100000:0.0:1769074959.236115:0:69562:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=7, status=0 req@ffff9b0365a3a300 x1855007617076608/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 592/584 e 0 to 0 dl 1769075014 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074959.236129:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000bd0:0xb:0x0] suppgids 0 -1: rc 0 00010000:00010000:0.0:1769074959.236161:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 2/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.236174:0:69562:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 2/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00010000:0.0:1769074959.236182:0:69562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caaf from it ffffb436423e7940 00010000:00010000:0.0:1769074959.236186:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 3/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.236193:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 2/0,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.236342:0:69562:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074959.236359:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074959.236368:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.236381:0:69562:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.236388:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074959.236392:0:69562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074959.236396:0:69562:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:0.0:1769074959.236401:0:69562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423e7be8 00010000:00010000:0.0:1769074959.236404:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.236410:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.236418:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074959.236426:0:69562:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=16408 00000080:00200000:0.0:1769074959.236436:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), inode mode 41ed mask 1 00000080:00200000:0.0:1769074959.236442:0:69562:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204e.sanity-flr, flags=16392 00000080:00200000:0.0:1769074959.236462:0:69562:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074959.236470:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074959.236476:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.236483:0:69562:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.236488:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074959.236492:0:69562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074959.236494:0:69562:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:0.0:1769074959.236499:0:69562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423e7b58 00010000:00010000:0.0:1769074959.236502:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.236509:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.236517:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074959.236521:0:69562:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=13 00000080:00200008:0.0:1769074959.236528:0:69562:0:(file.c:6252:ll_getattr_dentry()) START file d204e.sanity-flr:[0x200000405:0x20d:0x0](ffff9b0355ce63c8), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074959.236536:0:69562:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8),name=d204e.sanity-flr 00000002:00010000:0.0:1769074959.236551:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20d:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074959.236557:0:69562:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 30 res 8589935621/525 (0 0) 00010000:00010000:0.0:1769074959.236593:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236599:0:69562:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.236607:0:69562:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00000100:00100000:0.0:1769074959.236627:0:69562:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076736:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074959.236711:0:69562:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61d80 going to sleep for 50 seconds 00000100:00100000:0.0:1769074959.241206:0:69562:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076736:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074959.241272:0:69562:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241290:0:69562:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241299:0:69562:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 4/1,0 mode: --/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241310:0:69562:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241321:0:69562:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241332:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241340:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 5/2,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241347:0:69562:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074959.241358:0:69562:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff9b0365a3a300 x1855007617076736/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 576/672 e 0 to 0 dl 1769075014 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074959.241372:0:69562:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241381:0:69562:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/525 (0 0) 00000080:00200000:0.0:1769074959.241387:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074959.241416:0:69562:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20d:0x0], updating i_size 4096 00000080:00010000:0.0:1769074959.241423:0:69562:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20d:0x0](ffff9b0355ce63c8) for lock 0xb7d6bf619879cac4 00000080:00010000:0.0:1769074959.241431:0:69562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20d:0x0](ffff9b0355ce63c8) 00000080:00010000:0.0:1769074959.241434:0:69562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879cac4 from it ffffb436423e7cb8 00010000:00010000:0.0:1769074959.241437:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241445:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919fa00/0xb7d6bf619879cac4 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20d:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561c9 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000002:00010000:0.0:1769074959.241476:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000bd0:0xb:0x0]) in obj [0x200000bd0:0xb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074959.241486:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 2/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241497:0:69562:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 2/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241505:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 3/1,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241511:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 2/0,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000002:00010000:0.0:1769074959.241520:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x240000405:0xb:0x0]) in obj [0x240000405:0xb:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074959.241526:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 2/1,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241534:0:69562:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 2/1,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241541:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 3/1,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.241548:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 2/0,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200008:0.0:1769074959.242093:0:69562:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:0.0:1769074959.242099:0:69562:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074959.242111:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074959.242119:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.242127:0:69562:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.242134:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074959.242139:0:69562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074959.242141:0:69562:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:0.0:1769074959.242146:0:69562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423e7cb8 00010000:00010000:0.0:1769074959.242149:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.242156:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.242298:0:69562:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074959.242307:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074959.242313:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.242320:0:69562:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.242325:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074959.242329:0:69562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074959.242332:0:69562:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:0.0:1769074959.242336:0:69562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423e7b58 00010000:00010000:0.0:1769074959.242338:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.242344:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.242351:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074959.242357:0:69562:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=24 00000080:00200000:0.0:1769074959.242361:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), inode mode 41ed mask 1 00000080:00200000:0.0:1769074959.242364:0:69562:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204e.sanity-flr, flags=8 00000080:00200000:0.0:1769074959.242377:0:69562:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:0.0:1769074959.242382:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:0.0:1769074959.242387:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.242394:0:69562:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.242400:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:0.0:1769074959.242403:0:69562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:0.0:1769074959.242405:0:69562:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:0.0:1769074959.242408:0:69562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423e7bf8 00010000:00010000:0.0:1769074959.242411:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.242418:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.242426:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:0.0:1769074959.242429:0:69562:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=25 00000080:00200000:0.0:1769074959.242432:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), inode mode 41ed mask 1 00000080:00200000:0.0:1769074959.242436:0:69562:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f204e.sanity-flr, flags=777 00000080:00200000:0.0:1769074959.242439:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), inode mode 41ed mask 3 00000080:00200000:0.0:1769074959.242443:0:69562:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), file ffff9b035115b680, open_flags a141, mode 81b6 opened 0 00000080:00200000:0.0:1769074959.242453:0:69562:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), intent=open|creat 00000002:00010000:0.0:1769074959.242470:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204e.sanity-flr,[0x200000405:0x20e:0x0]) in obj [0x200000bd0:0xb:0x0], intent: open|creat flags 0120502 00010000:00010000:0.0:1769074959.242481:0:69562:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 2/0,0 mode: PR/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.242528:0:69562:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 1/0,0 mode: --/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.242534:0:69562:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1769074959.242537:0:69562:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860600/0xb7d6bf619879caaf lrc: 0/0,0 mode: --/PR res: [0x200000bd0:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd56198 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.242567:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 3/0,1 mode: --/CW res: [0x200000bd0:0xb:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.242573:0:69562:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 3/0,1 mode: --/CW res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00000020:00100000:0.0:1769074959.242584:0:69562:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074959.242594:0:69562:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a3a300 x1855007617076992/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 536/67824 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074959.242603:0:69562:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 3/0,1 mode: --/CW res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00000100:00100000:0.0:1769074959.242623:0:69562:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076992:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074959.242744:0:69562:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61d80 going to sleep for 50 seconds 00000100:00100000:0.0:1769074959.262988:0:69562:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3a300 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617076992:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074959.263015:0:69562:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 4/0,1 mode: --/CW res: [0x200000bd0:0xb:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.263046:0:69562:0:(ldlm_request.c:717:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 4/0,1 mode: --/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.263054:0:69562:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 4/0,1 mode: --/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.263065:0:69562:0:(ldlm_lock.c:1065:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 4/0,1 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.263075:0:69562:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 4/0,1 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.263086:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 5/1,1 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.263093:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 5/1,1 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.263101:0:69562:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 4/1,0 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769074959.263111:0:69562:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=3 disposition=8000037, status=0 req@ffff9b0365a3a300 x1855007617076992/t8589944146(8589944146) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 536/608 e 0 to 0 dl 1769075014 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1769074959.263124:0:69562:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0365a3a300 x1855007617076992/t8589944146(8589944146) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 536/608 e 0 to 0 dl 1769075014 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:0.0:1769074959.263136:0:69562:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.263145:0:69562:0:(ldlm_lock.c:1532:ldlm_lock_match_with_skip()) ### not matched ns ffff9b0346213800 type 13 mode 4 res 8589935621/526 (0 0) 00000080:00200000:0.0:1769074959.263152:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 3 on i1 [0x200000bd0:0xb:0x0] suppgids 0 -1: rc 0 00000080:00200000:0.0:1769074959.263165:0:69562:0:(namei.c:1569:ll_create_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), intent=open|creat 00000080:00200000:0.0:1769074959.263207:0:69562:0:(llite_lib.c:3217:ll_read_inode2()) VFS Op:inode=[0x200000405:0x20e:0x0](ffff9b0355e80908) 00000080:00200000:0.0:1769074959.263213:0:69562:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20e:0x0], updating i_size 0 00000080:00200000:0.0:1769074959.263294:0:69562:0:(file.c:6809:ll_layout_conf()) [0x200000405:0x20e:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:0.0:1769074959.263300:0:69562:0:(namei.c:1535:ll_create_node()) setting l_ast_data to inode [0x200000405:0x20e:0x0](ffff9b0355e80908) 00000080:00010000:0.0:1769074959.263303:0:69562:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000405:0x20e:0x0](ffff9b0355e80908) for lock 0xb7d6bf619879cacb 00000080:00010000:0.0:1769074959.263313:0:69562:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000405:0x20e:0x0](ffff9b0355e80908) 00000080:00200008:0.0:1769074959.263327:0:69562:0:(file.c:1001:ll_file_open()) START file f204e.sanity-flr:[0x200000405:0x20e:0x0](ffff9b0355e80908), flags 120501 00000002:00100000:0.0:1769074959.263346:0:69562:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0365a3a300 x1855007617076992/t8589944146(8589944146) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 536/608 e 0 to 0 dl 1769075014 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00010000:0.0:1769074959.263382:0:69562:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879cacb from it ffff9b0348361a20 00010000:00010000:0.0:1769074959.263386:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 3/1,0 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.263395:0:69562:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:0.0:1769074959.263414:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20e:0x0](ffff9b0355e80908), inode mode 81a4 mask 40 00000080:00200000:0.0:1769074959.263461:0:69562:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20e:0x0](ffff9b0355e80908), inode mode 81a4 mask 2 00000080:00200000:0.0:1769074959.263470:0:69562:0:(xattr.c:406:ll_xattr_set()) VFS Op:inode=[0x200000405:0x20e:0x0](ffff9b0355e80908), xattr lov 00000002:00010000:0.0:1769074959.263484:0:69562:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20e:0x0]) in obj [0x200000405:0x20e:0x0], intent: open flags 040000002 00010000:00010000:0.0:1769074959.263527:0:69562:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.263533:0:69562:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00000020:00100000:0.0:1769074959.263543:0:69562:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:0.0:1769074959.263549:0:69562:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a39880 x1855007617077248/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:0.0:1769074959.263558:0:69562:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 3/1,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00000100:00100000:0.0:1769074959.263575:0:69562:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617077248:192.168.204.160@tcp:101:lfs.0 00000100:00100000:0.0:1769074959.263747:0:69562:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61d80 going to sleep for 50 seconds 00000100:00100000:1.0:1769074959.268598:0:2398:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.160@tcp (source: 12345-192.168.204.160@tcp) 00000100:00100000:0.0:1769074959.268694:0:55997:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:0.0:1769074959.268713:0:55997:0:(service.c:2279:ptlrpc_server_handle_req_in()) unwrap req x1855007625298048 00000100:00100000:0.0:1769074959.268723:0:55997:0:(service.c:2344:ptlrpc_server_handle_req_in()) got req x1855007625298048 00000100:00100000:0.0:1769074959.268782:0:55997:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.160@tcp, seq: 8970 00000100:00100000:0.0:1769074959.268792:0:55997:0:(service.c:2530:ptlrpc_server_handle_request()) Handling RPC req@ffff9b0348c2fc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6512:x1855007625298048:12345-192.168.204.160@tcp:104: 00010000:00010000:0.0:1769074959.268808:0:55997:0:(ldlm_lockd.c:2505:ldlm_callback_handler()) ### blocking ast ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769074959.268884:0:55997:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1769074959.268891:0:55997:0:(service.c:2588:ptlrpc_server_handle_request()) Handled RPC req@ffff9b0348c2fc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:249ac4ab-c06c-4166-99d5-c4c7e1cc2791+5:6512:x1855007625298048:12345-192.168.204.160@tcp:104: Request processed in 102us (305us total) trans 0 rc 0/0 00000100:00100000:0.0:1769074959.268899:0:55997:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.160@tcp, seq: 8970 00010000:00010000:3.0:1769074959.268959:0:57730:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769074959.268980:0:57730:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x420000000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.268991:0:57730:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919e200 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:3.0:1769074959.268996:0:57730:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.269004:0:57730:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x420400000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.269013:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 3/0,0 mode: PR/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x428400000000 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.269042:0:57730:0:(vvp_object.c:114:vvp_conf_set()) [0x200000405:0x20e:0x0]: losing layout lock 00010000:00010000:3.0:1769074959.269085:0:57730:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 2/0,0 mode: --/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.269092:0:57730:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1769074959.269126:0:57730:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 1/0,0 mode: --/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.269134:0:57730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e200/0xb7d6bf619879cacb lrc: 0/0,0 mode: --/PR res: [0x200000405:0x20e:0x0].0x0 bits 0x1b/0x1b rrc: 3 type: IBT gid 0 flags: 0x4c29400000020 nid: local remote: 0x7fe2323adfd561e5 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.269147:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074959.269148:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074959.269150:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074959.269219:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617077376:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1769074959.273587:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617077376:192.168.204.160@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1769074959.280496:0:69562:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39880 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617077248:192.168.204.160@tcp:101:lfs.0 00010000:00010000:0.0:1769074959.280521:0:69562:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 4/1,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.280535:0:69562:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 4/1,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.280541:0:69562:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 4/1,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.280548:0:69562:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 3/0,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.280555:0:69562:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 4/0,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.280561:0:69562:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919e600 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:0.0:1769074959.280566:0:69562:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 4/0,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.280572:0:69562:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 4/0,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.280578:0:69562:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 5/0,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.280584:0:69562:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1769074959.280589:0:69562:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 3/0,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 69562 initiator: MDT0 00010000:00010000:0.0:1769074959.280595:0:69562:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919e600/0xb7d6bf619879cad2 lrc: 0/0,0 mode: --/CR res: [0x200000405:0x20e:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 69562 initiator: MDT0 00000002:00100000:0.0:1769074959.280636:0:69562:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=800002b, status=0 req@ffff9b0365a39880 x1855007617077248/t8589944147(8589944147) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/1056 e 0 to 0 dl 1769075014 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1769074959.280655:0:69562:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0365a39880 x1855007617077248/t8589944147(8589944147) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/1056 e 0 to 0 dl 1769075014 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:0.0:1769074959.280673:0:69562:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000405:0x20e:0x0] suppgids 0 0: rc 0 00000080:00200000:0.0:1769074959.280722:0:69562:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x20e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769074959.280732:0:69562:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x20e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1769074959.280778:0:69562:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20e:0x0], updating i_size 0 00000002:00100000:0.0:1769074959.280805:0:69562:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0365a39880 x1855007617077248/t8589944147(8589944147) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 968/1056 e 0 to 0 dl 1769075014 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:0.0:1769074959.280849:0:69562:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0365a39880 x1855007617077248/t8589944147(8589944147) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 968/1056 e 0 to 0 dl 1769075014 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074959.280858:0:69562:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074959.280863:0:69562:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a38380 x1855007617077632/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074959.280882:0:69562:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617077632:192.168.204.160@tcp:35:lfs.0 00000100:00100000:0.0:1769074959.280964:0:69562:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61d80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074959.283548:0:69562:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38380 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617077632:192.168.204.160@tcp:35:lfs.0 00000002:00100000:0.0:1769074959.283586:0:69562:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0365a39880 x1855007617077248/t8589944147(8589944147) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 968/1056 e 0 to 0 dl 1769075014 ref 2 fl Complete:RQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:0.0:1769074959.283652:0:69562:0:(file.c:424:ll_file_release()) START file f204e.sanity-flr:[0x200000405:0x20e:0x0](ffff9b0355e80908), flags 100001 00000002:00100000:0.0:1769074959.283702:0:69562:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0365a3a300 x1855007617076992/t8589944146(8589944146) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 536/608 e 0 to 0 dl 1769075014 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769074959.283712:0:69562:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:0.0:1769074959.283717:0:69562:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a38700 x1855007617077888/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:0.0:1769074959.283737:0:69562:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617077888:192.168.204.160@tcp:35:lfs.0 00000100:00100000:0.0:1769074959.283952:0:69562:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61d80 going to sleep for 11 seconds 00000100:00100000:0.0:1769074959.308749:0:69562:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38700 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69562:1855007617077888:192.168.204.160@tcp:35:lfs.0 00000002:00100000:0.0:1769074959.309081:0:69562:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0365a3a300 x1855007617076992/t8589944146(8589944146) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 536/608 e 0 to 0 dl 1769075014 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1769074959.349904:0:69563:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:3.0:1769074959.349937:0:69563:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074959.349955:0:69563:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.349992:0:69563:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.350002:0:69563:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074959.350010:0:69563:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:3.0:1769074959.350014:0:69563:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:3.0:1769074959.350021:0:69563:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423efbf8 00010000:00010000:3.0:1769074959.350025:0:69563:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.350039:0:69563:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.350056:0:69563:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:3.0:1769074959.350064:0:69563:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:3.0:1769074959.350071:0:69563:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), inode mode 41ed mask 1 00000080:00200000:3.0:1769074959.350084:0:69563:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), file ffff9b03462a3200, open_flags 18800, mode 0 opened 0 00000080:00200000:3.0:1769074959.350091:0:69563:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), intent=open 00000002:00010000:3.0:1769074959.350108:0:69563:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0xb:0x0], intent: open flags 0304001 00010000:00010000:3.0:1769074959.350164:0:69563:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.350172:0:69563:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69563 initiator: MDT0 00000020:00100000:3.0:1769074959.350183:0:69563:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1769074959.350196:0:69563:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0352a18e00 x1855007617078144/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1769074959.350206:0:69563:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69563 initiator: MDT0 00000100:00100000:3.0:1769074959.350227:0:69563:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69563:1855007617078144:192.168.204.160@tcp:101:lfs.0 00000100:00100000:3.0:1769074959.350662:0:69563:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a180 going to sleep for 50 seconds 00000100:00100000:3.0:1769074959.354811:0:69563:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69563:1855007617078144:192.168.204.160@tcp:101:lfs.0 00010000:00010000:3.0:1769074959.354836:0:69563:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.354853:0:69563:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.354859:0:69563:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 4/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.354866:0:69563:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x10206400000000 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.354873:0:69563:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x206400000000 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.354879:0:69563:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919de00 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:3.0:1769074959.354883:0:69563:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.354889:0:69563:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 4/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x206400000000 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.354896:0:69563:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 5/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x20e400000000 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.354902:0:69563:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1769074959.354908:0:69563:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 3/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.354914:0:69563:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919de00/0xb7d6bf619879cad9 lrc: 0/0,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x1 rrc: 2 type: IBT flags: 0x4a0f400000000 pid: 69563 initiator: MDT0 00000002:00100000:3.0:1769074959.354938:0:69563:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=b, status=-20 req@ffff9b0352a18e00 x1855007617078144/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/608 e 0 to 0 dl 1769075014 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1769074959.354953:0:69563:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000bd0:0xb:0x0] suppgids 0 -1: rc -20 00000080:00200000:3.0:1769074959.355112:0:69563:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:3.0:1769074959.355133:0:69563:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1769074959.355147:0:69563:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.355158:0:69563:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.355165:0:69563:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1769074959.355171:0:69563:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:3.0:1769074959.355175:0:69563:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:3.0:1769074959.355180:0:69563:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423efbf8 00010000:00010000:3.0:1769074959.355183:0:69563:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.355192:0:69563:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:3.0:1769074959.355205:0:69563:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:3.0:1769074959.355213:0:69563:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=267 00000080:00200000:3.0:1769074959.355219:0:69563:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), inode mode 41ed mask 44 00000080:00200008:3.0:1769074959.355228:0:69563:0:(file.c:1001:ll_file_open()) START file d204e.sanity-flr:[0x200000405:0x20d:0x0](ffff9b0355ce63c8), flags 304000 00000002:00010000:3.0:1769074959.355560:0:69563:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000405:0x20d:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240304001 00010000:00010000:3.0:1769074959.355626:0:69563:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 69563 initiator: MDT0 00010000:00010000:3.0:1769074959.355634:0:69563:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69563 initiator: MDT0 00000020:00100000:3.0:1769074959.355645:0:69563:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1769074959.355654:0:69563:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0352a18e00 x1855007617078400/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1769074959.355663:0:69563:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69563 initiator: MDT0 00000100:00100000:3.0:1769074959.355684:0:69563:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69563:1855007617078400:192.168.204.160@tcp:101:lfs.0 00000100:00100000:3.0:1769074959.356201:0:69563:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0372c1a180 going to sleep for 50 seconds 00000100:00100000:2.0:1769074959.362578:0:69563:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69563:1855007617078400:192.168.204.160@tcp:101:lfs.0 00010000:00010000:2.0:1769074959.362657:0:69563:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 69563 initiator: MDT0 00010000:00010000:2.0:1769074959.362671:0:69563:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69563 initiator: MDT0 00010000:00010000:2.0:1769074959.362678:0:69563:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69563 initiator: MDT0 00010000:00010000:2.0:1769074959.362686:0:69563:0:(ldlm_lock.c:880:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 69563 initiator: MDT0 00010000:00010000:2.0:1769074959.362694:0:69563:0:(ldlm_lockd.c:1906:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 69563 initiator: MDT0 00010000:00010000:2.0:1769074959.362699:0:69563:0:(ldlm_lockd.c:1920:ldlm_handle_bl_callback()) Lock ffff9b035919c600 already unused, calling callback (ffffffffc1af1c80) 00010000:00010000:2.0:1769074959.362704:0:69563:0:(ldlm_request.c:1595:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69563 initiator: MDT0 00010000:00010000:2.0:1769074959.362711:0:69563:0:(ldlm_request.c:1605:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 69563 initiator: MDT0 00010000:00010000:2.0:1769074959.362718:0:69563:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 69563 initiator: MDT0 00010000:00010000:2.0:1769074959.362725:0:69563:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1769074959.362731:0:69563:0:(ldlm_lockd.c:1932:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69563 initiator: MDT0 00010000:00010000:2.0:1769074959.362737:0:69563:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035919c600/0xb7d6bf619879cae0 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 69563 initiator: MDT0 00000002:00100000:2.0:1769074959.362755:0:69563:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=1 disposition=2b, status=0 req@ffff9b0352a18e00 x1855007617078400/t8589944150(8589944150) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769075014 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1769074959.362768:0:69563:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0352a18e00 x1855007617078400/t8589944150(8589944150) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2152/696 e 0 to 0 dl 1769075014 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1769074959.362868:0:69563:0:(namei.c:955:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1769074959.362906:0:69563:0:(llite_lib.c:2981:ll_update_inode()) inode=[0x200000405:0x20d:0x0], updating i_size 4096 00000002:00100000:2.0:1769074959.362924:0:69563:0:(mdc_request.c:810:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9b0352a18e00 x1855007617078400/t8589944150(8589944150) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769075014 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200080:2.0:1769074959.363136:0:69563:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8) cmd=c0086915 arg=bf43d0 00000100:00100000:2.0:1769074959.363202:0:69563:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69563:1855007617078528:192.168.204.160@tcp:34:lfs.0 00000100:00100000:2.0:1769074959.363577:0:69563:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ed80 going to sleep for 50 seconds 00000100:00100000:2.0:1769074959.367762:0:69563:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69563:1855007617078528:192.168.204.160@tcp:34:lfs.0 00000080:00200008:2.0:1769074959.368332:0:69563:0:(file.c:424:ll_file_release()) START file d204e.sanity-flr:[0x200000405:0x20d:0x0](ffff9b0355ce63c8), flags 304000 00000002:00100000:2.0:1769074959.368400:0:69563:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0352a18e00 x1855007617078400/t8589944150(8589944150) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769075014 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769074959.368413:0:69563:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1769074959.368419:0:69563:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c6680 x1855007617078784/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1769074959.368440:0:69563:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69563:1855007617078784:192.168.204.160@tcp:35:lfs.0 00000100:00100000:2.0:1769074959.368530:0:69563:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ed80 going to sleep for 11 seconds 00000100:00100000:2.0:1769074959.371580:0:69563:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69563:1855007617078784:192.168.204.160@tcp:35:lfs.0 00000002:00100000:2.0:1769074959.371662:0:69563:0:(mdc_request.c:831:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9b0352a18e00 x1855007617078400/t8589944150(8589944150) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769075014 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200008:1.0:1769074959.410066:0:69567:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074959.410076:0:69567:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074959.410098:0:69567:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074959.410114:0:69567:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074959.410130:0:69567:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074959.410138:0:69567:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074959.410145:0:69567:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074959.410149:0:69567:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:1.0:1769074959.410156:0:69567:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423ffcb8 00010000:00010000:1.0:1769074959.410160:0:69567:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074959.410169:0:69567:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074959.410473:0:69567:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074959.410482:0:69567:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074959.410490:0:69567:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074959.410498:0:69567:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074959.410504:0:69567:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074959.410508:0:69567:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074959.410511:0:69567:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:1.0:1769074959.410514:0:69567:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423ffbb0 00010000:00010000:1.0:1769074959.410517:0:69567:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074959.410524:0:69567:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074959.410533:0:69567:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074959.410540:0:69567:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200080:1.0:1769074959.410558:0:69567:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=800866a8 arg=7ffc529bb80c 00000080:00200008:1.0:1769074959.410573:0:69567:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 300000 00000080:00200000:1.0:1769074959.410593:0:69567:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074959.410600:0:69567:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074959.410606:0:69567:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074959.410614:0:69567:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074959.410621:0:69567:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074959.410624:0:69567:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074959.410627:0:69567:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:1.0:1769074959.410630:0:69567:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423ffbb0 00010000:00010000:1.0:1769074959.410633:0:69567:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074959.410640:0:69567:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074959.410648:0:69567:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 44 00000080:00200008:1.0:1769074959.410651:0:69567:0:(file.c:1001:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 100000 00000080:00200080:1.0:1769074959.410658:0:69567:0:(dir.c:2017:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88) cmd=8080667f arg=7ffc529ba4e0 00000080:00200008:1.0:1769074959.411431:0:69567:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), flags 100000 00000080:00200008:1.0:1769074959.411765:0:69567:0:(file.c:6252:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9b0352593a88), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1769074959.411770:0:69567:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:1.0:1769074959.413442:0:69567:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1769074959.413465:0:69567:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074959.413481:0:69567:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:1.0:1769074959.413490:0:69567:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1769074959.413499:0:69567:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:1.0:1769074959.413503:0:69567:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:1.0:1769074959.413510:0:69567:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423ffcb8 00010000:00010000:1.0:1769074959.413515:0:69567:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:1.0:1769074959.413523:0:69567:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:3.0:1769074959.827920:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074960.852175:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074961.876605:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074962.004908:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617079040:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074962.005064:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617079296:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074962.005099:0:2409:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1769074962.005106:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2409:1855007617078912:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074962.005130:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617079168:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074962.005147:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617079424:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074962.010120:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617079168:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074962.010142:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617079040:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074962.010153:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617079296:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074962.011950:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2409:1855007617078912:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074962.012818:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617079424:192.168.204.160@tcp:400:kworker.0 00010000:00010000:1.0:1769074962.902720:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074963.923490:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074964.947781:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074965.972223:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074966.995657:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769074967.123557:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617079552:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074967.123578:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617079680:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074967.123658:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617079936:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074967.123659:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617079808:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074967.123702:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617080064:192.168.204.160@tcp:400:kworker.0 00010000:00010000:2.0:1769074968.019489:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074969.043467:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074970.067454:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074971.091427:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1769074971.257358:0:69591:0:(file.c:6147:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88),name=/ 00000002:00010000:2.0:1769074971.257391:0:69591:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1769074971.257408:0:69591:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074971.257422:0:69591:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074971.257428:0:69591:0:(namei.c:955:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1769074971.257435:0:69591:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) 00000080:00010000:2.0:1769074971.257438:0:69591:0:(llite_internal.h:2004:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9b0352593a88) for lock 0xb7d6bf619879caa1 00000080:00010000:2.0:1769074971.257444:0:69591:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb7d6bf619879caa1 from it ffffb436423efbf8 00010000:00010000:2.0:1769074971.257447:0:69591:0:(ldlm_lock.c:822:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769074971.257453:0:69591:0:(ldlm_lock.c:905:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860a00/0xb7d6bf619879caa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd5616e expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00000080:00200000:2.0:1769074971.257467:0:69591:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9b0352593a88), inode mode 41ed mask 1 00000080:00200000:2.0:1769074971.257473:0:69591:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d204e.sanity-flr, flags=27 00000080:00200000:2.0:1769074971.257478:0:69591:0:(file.c:6629:ll_inode_permission()) VFS Op:inode=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), inode mode 41ed mask 1 00000080:00200000:2.0:1769074971.257487:0:69591:0:(namei.c:1354:ll_atomic_open()) VFS Op:name=f204e.sanity-flr, dir=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), file ffff9b0347d28180, open_flags 18800, mode 0 opened 0 00000080:00200000:2.0:1769074971.257492:0:69591:0:(namei.c:1020:ll_lookup_it()) VFS Op:name=f204e.sanity-flr, dir=[0x200000405:0x20d:0x0](ffff9b0355ce63c8), intent=open 00000002:00010000:2.0:1769074971.257508:0:69591:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: f204e.sanity-flr,[0x0:0x0:0x0]) in obj [0x200000bd0:0xb:0x0], intent: open flags 0304001 00010000:00010000:2.0:1769074971.257570:0:69591:0:(ldlm_lock.c:768:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879cae7 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 69591 initiator: MDT0 00010000:00010000:2.0:1769074971.257578:0:69591:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879cae7 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69591 initiator: MDT0 00000020:00100000:2.0:1769074971.257591:0:69591:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1769074971.257607:0:69591:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7480 x1855007617080320/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1769074971.257617:0:69591:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b0365860400/0xb7d6bf619879cae7 lrc: 3/1,0 mode: --/CR res: [0x200000bd0:0xb:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 69591 initiator: MDT0 00000100:00100000:2.0:1769074971.257644:0:69591:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7480 pname:cluuid:pid:xid:nid:opc:job lfs:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:69591:1855007617080320:192.168.204.160@tcp:101:lfs.0 00000100:00100000:2.0:1769074971.257854:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 50 seconds 00010000:00010000:0.0:1769074972.115422:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769074972.243490:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617080576:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074972.243490:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2409:1855007617080448:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074972.243611:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617080704:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074972.243612:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617080832:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074972.243654:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617080960:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769074973.139408:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074974.163448:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074975.187391:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074976.211445:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074977.235367:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074977.363493:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617081088:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074977.363498:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617081216:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074977.363601:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617081472:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074977.363602:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617081344:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074977.363655:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ce9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617081600:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769074978.259443:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074979.283428:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074980.307446:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074981.331397:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074982.355438:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769074982.483595:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617081728:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074982.483715:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617081984:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074982.483756:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617082240:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074982.483787:0:2406:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1769074982.483795:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617081856:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074982.483821:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617082112:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769074983.379450:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769074983.507356:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074967/real 1769074967] req@ffff9b0342ce9180 x1855007617079552/t0(0) o400->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074983 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:0.0:1769074983.507358:0:2407:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074967/real 1769074967] req@ffff9b0342cea680 x1855007617079936/t0(0) o400->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074983 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074983.507418:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342ce9180 x1855007617079552/t0(0) o400->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074983 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074983.507432:0:2406:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1769074983.507436:0:2406:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.160@tcp: Connection to MGS (at 192.168.204.160@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1769074983.519399:0:2406:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.160@tcp for MGC192.168.204.160@tcp not replayable, auto-deactivating 00000100:00080000:1.0:1769074983.519403:0:2406:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:1.0:1769074983.519412:0:2406:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff9b0342ce9180 x1855007617079552/t0(0) o400->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 224/224 e 0 to 1 dl 1769074983 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074983.519422:0:2406:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff9b03482c6680 x1855007617080448/t0(0) o400->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074988 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074983.519435:0:2406:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff9b0342ce8e00 x1855007617081088/t0(0) o400->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074993 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074983.519442:0:2406:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff9b0365a38000 x1855007617081728/t0(0) o400->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 224/224 e 0 to 0 dl 1769074998 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:1.0:1769074983.519449:0:2406:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1769074983.519451:0:2406:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1769074983.519452:0:2406:0:(mgc_request.c:1165:mgc_import_event()) import event 0x808002 00000100:00080000:1.0:1769074983.519455:0:2406:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00080000:1.0:1769074983.519483:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074983.519487:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769074983.519490:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769074983.519497:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 1223 00000100:00080000:1.0:1769074983.519514:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769074983.519543:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2409:1855007617080448:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074983.519553:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0342ceb480 x1855007617082368/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074983.519639:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617082368:192.168.204.160@tcp:250:kworker.0 00000100:00000400:1.0:1769074983.519642:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074967/real 1769074967] req@ffff9b0342ceb100 x1855007617080064/t0(0) o400->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074983 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074983.519665:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342ceb100 x1855007617080064/t0(0) o400->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074983 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074983.519673:0:2406:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1769074983.519676:0:2406:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffff9b0342ad2800: Connection to lustre-OST0001 (at 192.168.204.160@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00100000:0.0:1769074983.521384:0:2407:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342cea680 x1855007617079936/t0(0) o400->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074983 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074983.521392:0:2407:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:0.0:1769074983.521395:0:2407:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff9b0342ad2800: Connection to lustre-OST0000 (at 192.168.204.160@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:0.0:1769074983.521398:0:2407:0:(import.c:416:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00080000:2.0:1769074983.521425:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074983.521430:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769074983.521432:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074983.521438:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 1058 00000100:00080000:2.0:1769074983.521447:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769074983.521458:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769074983.521467:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617082496:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074983.521478:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617083008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769074983.521512:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:0:1855007617082752:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074983.521520:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7835f93-1e5d-43a4-8110-91f072d9ae65:2407:1855007617081088:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769074983.521529:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617079936:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074983.521532:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617083008:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769074983.521545:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:0.0:1769074983.521552:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617082624:192.168.204.160@tcp:400:kworker.0 00000100:00080000:1.0:1769074983.527373:0:2406:0:(import.c:416:ptlrpc_pinger_force()) lustre-OST0001_UUID: waking up pinger s:DISCONN 00000100:00080000:0.0:1769074983.527377:0:2407:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9b0343dee680 x1855007617082880/t0(0) o400->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl Rpc:NWQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074983.527436:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074983.527441:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769074983.527444:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074983.527449:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 1817 00000100:00080000:2.0:1769074983.527459:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769074983.527465:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769074983.527486:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617081728:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074983.527487:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617083136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769074983.527498:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7835f93-1e5d-43a4-8110-91f072d9ae65:2406:1855007617079552:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074983.527539:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617083136:192.168.204.160@tcp:8:kworker.0 00010000:00010000:1.0:1769074984.403416:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074985.427428:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074986.451372:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074987.475418:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074988.499395:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769074988.627347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074988.627358:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:3.0:1769074988.627388:0:2409:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074972/real 1769074972] req@ffff9b03482c5f80 x1855007617080960/t0(0) o400->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074988 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074988.627408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074988.627412:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1769074988.627414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074988.627418:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:2.0:1769074988.627436:0:2408:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074972/real 1769074972] req@ffff9b03482c4e00 x1855007617080832/t0(0) o400->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074988 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769074988.627459:0:2408:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b03482c4e00 x1855007617080832/t0(0) o400->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074988 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769074988.627465:0:2408:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff9b0352ab1000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1769074988.627498:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617083264:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074988.627626:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617080832:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769074988.627649:0:2408:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1769074988.627657:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617083392:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074988.635665:0:2409:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b03482c5f80 x1855007617080960/t0(0) o400->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074988 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074988.635673:0:2409:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff9b0352ab7000 already not connected (conn 3, was 2): CONNECTING 00010000:00010000:2.0:1769074989.523489:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074990.547428:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074991.571443:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074992.595399:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769074993.619442:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769074993.683369:0:2407:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074977/real 1769074977] req@ffff9b0342ce9880 x1855007617081600/t0(0) o400->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074993.691184:0:2407:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342ce9880 x1855007617081600/t0(0) o400->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074993.691194:0:2407:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff9b0352ab7000 already not connected (conn 3, was 2): CONNECTING 00000100:00000400:0.0:1769074993.747366:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074977/real 1769074977] req@ffff9b0342ce9500 x1855007617081472/t0(0) o400->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074993.747425:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342ce9500 x1855007617081472/t0(0) o400->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769074993.747439:0:2406:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff9b0352ab1000 already not connected (conn 2, was 1): CONNECTING 00000100:00080000:1.0:1769074993.747439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074993.747455:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1769074993.747479:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617081472:192.168.204.160@tcp:400:kworker.0 00000100:00080000:1.0:1769074993.747507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074993.747512:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1769074993.747515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769074993.747519:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1769074993.747528:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074983/real 1769074983] req@ffff9b03482c5880 x1855007617083136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769074993.747542:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617083648:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074993.747555:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617083136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074993.747566:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074983/real 1769074983] req@ffff9b03482c5c00 x1855007617083008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074993.747570:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617083520:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074993.747586:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617083008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074993.747594:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074983/real 1769074983] req@ffff9b0342ceb480 x1855007617082368/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769074993.747609:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0342ceb480 x1855007617082368/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769074993 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074993.747641:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769074993.747671:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769074993.747683:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617083136:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769074993.747687:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:0:1855007617082880:192.168.204.160@tcp:400:kworker.0 00000100:00080000:3.0:1769074993.747698:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769074993.747700:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617080960:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074993.747702:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617081600:192.168.204.160@tcp:400:kworker.0 00000100:00080000:3.0:1769074993.747718:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769074993.747730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617083008:192.168.204.160@tcp:8:kworker.0 00000100:00080000:3.0:1769074993.747745:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769074993.747768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769074993.747777:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617082368:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769074993.747800:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074993.747806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769074993.747809:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769074993.747814:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617080064:192.168.204.160@tcp:400:kworker.0 00000100:00080000:2.0:1769074993.747821:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 1977 00000100:00080000:2.0:1769074993.747831:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1769074993.747840:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769074993.747866:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617083776/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074993.747892:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074993.747896:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769074993.747898:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769074993.747903:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 1977 00000100:00080000:2.0:1769074993.747911:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1769074993.747917:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769074993.747924:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617083776:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769074993.747929:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769074993.747953:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617083904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769074993.747970:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769074993.747975:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769074993.747979:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769074993.747983:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617083904:192.168.204.160@tcp:8:kworker.0 00000100:00080000:2.0:1769074993.747987:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 1977 00000100:00080000:2.0:1769074993.747997:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1769074993.748005:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769074993.748015:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769074993.748034:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617084032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769074993.748089:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617084032:192.168.204.160@tcp:8:kworker.0 00010000:00010000:3.0:1769074994.643502:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074995.667471:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769074996.691443:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769074997.715435:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769074998.739401:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769074998.867367:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074982/real 1769074982] req@ffff9b0365a38a80 x1855007617082112/t0(0) o400->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074998 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769074998.867427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074998.867433:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769074998.867509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074998.867514:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769074998.867516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769074998.867519:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1769074998.867553:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617084160:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769074998.867556:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617084288:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769074998.876511:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0365a38a80 x1855007617082112/t0(0) o400->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074998 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074998.876521:0:2406:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff9b0352ab1000 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:1.0:1769074998.876528:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074982/real 1769074982] req@ffff9b0365a3b800 x1855007617082240/t0(0) o400->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074998 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769074998.876538:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0365a3b800 x1855007617082240/t0(0) o400->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 224/224 e 0 to 1 dl 1769074998 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769074998.876543:0:2406:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff9b0352ab7000 already not connected (conn 4, was 2): CONNECTING 00000100:00100000:1.0:1769074998.876551:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617082112:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769074999.763401:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075000.787395:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075001.811449:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075002.835415:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075003.859457:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075003.987401:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075003.987417:0:59411:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1769075003.987496:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075003.987499:0:59411:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1769075003.987502:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075003.987506:0:59411:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1769075003.987544:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617084544:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769075003.987545:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617084416:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769075004.883424:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075005.907388:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075006.931441:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075007.955331:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075008.083346:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074993/real 1769074993] req@ffff9b03482c7b80 x1855007617084032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075008.083377:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617084032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075008.083386:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074993/real 1769074993] req@ffff9b03482c4e00 x1855007617083904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075008.083398:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617083904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075008.083408:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074993/real 1769074993] req@ffff9b03482c5f80 x1855007617083776/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075008.083420:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617083776/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075008 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075008.083466:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075008.083480:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075008.083483:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617084032:192.168.204.160@tcp:8:kworker.0 00000100:00080000:3.0:1769075008.083489:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075008.083493:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075008.083496:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617083904:192.168.204.160@tcp:8:kworker.0 00000100:00080000:3.0:1769075008.083503:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075008.083509:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075008.083512:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617083776:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075008.083521:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075008.083530:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00100000:1.0:1769075008.083531:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617082240:192.168.204.160@tcp:400:kworker.0 00000100:00080000:0.0:1769075008.083534:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075008.083546:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 1988 00000100:00080000:0.0:1769075008.083557:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1769075008.083565:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075008.083598:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b800 x1855007617084672/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075008.083644:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075008.083649:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1769075008.083649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617084672:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075008.083653:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075008.083658:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 1988 00000100:00080000:0.0:1769075008.083662:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1769075008.083666:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075008.083675:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075008.083676:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617084800:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769075008.083694:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617085056/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075008.083707:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075008.083710:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075008.083713:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075008.083718:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 1988 00000100:00080000:0.0:1769075008.083723:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1769075008.083728:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075008.083734:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075008.083747:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617085056:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075008.083750:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617085184/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075008.083759:0:2407:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1769075008.083773:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617084928:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769075008.083784:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617085184:192.168.204.160@tcp:8:kworker.0 00010000:00010000:3.0:1769075008.979438:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075010.003392:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075011.027446:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075012.051496:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075013.075407:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075013.203307:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075013.203316:0:59411:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769075013.203376:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075013.203379:0:59411:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1769075013.203381:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075013.203383:0:59411:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1769075013.203405:0:2406:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617085440:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769075013.203420:0:2407:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617085312:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769075014.099379:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075015.123405:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075016.147416:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1769075017.171364:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1769075018.195425:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075018.323321:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075018.323333:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075018.323405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075018.323412:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075018.323415:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075018.323419:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1769075018.323425:0:2408:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617085568:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769075018.323449:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617085696:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769075019.219403:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075020.243410:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.1:1769075020.883463:0:2397:0:(socklnd_cb.c:2407:ksocknal_find_timed_out_conn()) Timeout sending data to 12345-192.168.204.160@tcp (192.168.204.160:988) the network or that node may be down. 00010000:00010000:3.0:1769075021.267465:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075021.907318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 1 seconds 00010000:00010000:0.0:1769075022.291368:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769075022.355324:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074967/real 1769074967] req@ffff9b0342ce8700 x1855007617079808/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075022 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:1.0:1769075022.355324:0:2407:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074967/real 1769074967] req@ffff9b0342cea300 x1855007617079680/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075022 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075022.355377:0:2407:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342cea300 x1855007617079680/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075022 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075022.355397:0:2407:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1769075022.355402:0:2407:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff9b0342ad2800: Connection to lustre-MDT0000 (at 192.168.204.160@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1769075022.369874:0:2407:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:1.0:1769075022.369936:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cea300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617079680:192.168.204.160@tcp:400:kworker.0 00000100:00080000:3.0:1769075022.369937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075022.369956:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075022.369960:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075022.369964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075022.369968:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075022.369977:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 1223 00000100:00080000:3.0:1769075022.369992:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075022.370004:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075022.370048:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617085824/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075022.370102:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075022.370107:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617085824:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075022.370108:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075022.370115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075022.370120:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1769075022.370142:0:2409:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1769075022.370171:0:2409:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617085952:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769075022.373265:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342ce8700 x1855007617079808/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075022 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075022.373276:0:2406:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:0.0:1769075022.373281:0:2406:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffff9b0342ad2800: Connection to lustre-MDT0001 (at 192.168.204.160@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:0.0:1769075022.373285:0:2406:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0001_UUID: waking up pinger s:DISCONN 00000100:00100000:0.0:1769075022.373307:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617079808:192.168.204.160@tcp:400:kworker.0 00000100:00080000:3.0:1769075022.373354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075022.373358:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075022.373361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075022.373363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075022.373365:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075022.373369:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 1058 00000100:00080000:3.0:1769075022.373377:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075022.373382:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075022.373397:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617086080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075022.373413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075022.373416:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075022.373418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075022.373420:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1769075022.373449:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086080:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075022.931343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 1 seconds 00010000:00010000:3.0:1769075023.315388:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075023.955347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 1 seconds 00010000:00010000:3.0:1769075024.339419:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075024.979319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 1 seconds 00010000:00010000:2.0:1769075025.363388:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1769075026.003375:0:69591:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074971/real 1769074971] req@ffff9b03482c7480 x1855007617080320/t0(0) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 2168/133360 e 0 to 1 dl 1769075026 ref 2 fl Rpc:XPQr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1769075026.003404:0:69591:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 10, was 9): CONNECTING 00000100:00100000:2.0:1769075026.003410:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075026.387443:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075027.027327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075027.411426:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1769075027.475349:0:2408:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074972/real 1769074972] req@ffff9b03482c6a00 x1855007617080576/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075027 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1769075027.475350:0:2409:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074972/real 1769074972] req@ffff9b03482c4700 x1855007617080704/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075027 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075027.475400:0:2408:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b03482c6a00 x1855007617080576/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075027 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075027.475409:0:2409:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b03482c4700 x1855007617080704/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075027 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075027.475410:0:2408:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 10, was 9): CONNECTING 00000100:00080000:2.0:1769075027.475424:0:2409:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:3.0:1769075027.475427:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617080576:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769075027.475442:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617080704:192.168.204.160@tcp:400:kworker.0 00000100:00080000:3.0:1769075027.539339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539350:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075027.539354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539358:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075027.539361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539365:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075027.539368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539372:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075027.539374:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539378:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000400:00000100:1.0:1769075027.539550:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.204.160:988 00000400:00000100:1.0:1769075027.539583:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075027.539596:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075027.539630:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 224 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769075027.539647:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075022/real 1769075027] req@ffff9b0352a1aa00 x1855007617085824/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1769075027.539657:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769075027.539673:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617085824/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075027.539684:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075027.539700:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00000400:2.0:1769075027.539700:0:2409:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075022/real 1769075027] req@ffff9b0352a1b480 x1855007617085952/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075027.539706:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617085824:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075027.539726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769075027.539731:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075022/real 1769075027] req@ffff9b0352a19c00 x1855007617086080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075027.539731:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1769075027.539738:0:2409:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0352a1b480 x1855007617085952/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075027.539738:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539748:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075027.539752:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075027.539754:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617086080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075027.539762:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2016 00000100:00080000:2.0:1769075027.539773:0:2409:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 2, was 1): CONNECTING 00000100:00080000:0.0:1769075027.539774:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075027.539775:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1769075027.539787:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075027.539789:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617085952:192.168.204.160@tcp:400:kworker.0 00000100:00080000:0.0:1769075027.539794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075027.539800:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086080:192.168.204.160@tcp:38:kworker.0 00000008:00100000:3.0:1769075027.539800:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075027.539847:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617086208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075027.539873:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075027.539878:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075027.539882:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2016 00000100:00080000:3.0:1769075027.539888:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1769075027.539893:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075027.539899:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075027.539900:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075027.539916:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617086336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075027.539931:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539935:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075027.539937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539940:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075027.539946:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539949:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075027.539951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539953:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1769075027.539956:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075027.539959:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000800:00000100:0.0:1769075027.539991:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075027.540018:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075027/real 1769075027] req@ffff9b0352a18700 x1855007617086208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075027.540031:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617086208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075027.540040:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075027.540044:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075027.540047:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075027.540066:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075027.540086:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075027.540101:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075027/real 1769075027] req@ffff9b0352a19f80 x1855007617086336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075027.540113:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617086336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075027.540121:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075027.540125:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075027.540128:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075028.051300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075028.435430:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075028.563416:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075008/real 1769075008] req@ffff9b0365a39500 x1855007617085184/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075028 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075028.563450:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617085184/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075028 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075028.563462:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075008/real 1769075008] req@ffff9b0365a39c00 x1855007617085056/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075028 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075028.563476:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617085056/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075028 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075028.563493:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075008/real 1769075008] req@ffff9b0365a3b800 x1855007617084672/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075028 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075028.563508:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b800 x1855007617084672/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075028 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075028.563522:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075028.563554:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075028.563559:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617085184:192.168.204.160@tcp:8:kworker.0 00000100:00080000:0.0:1769075028.563565:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075028.563572:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075028.563576:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617085056:192.168.204.160@tcp:8:kworker.0 00000100:00080000:0.0:1769075028.563581:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075028.563586:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075028.563589:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617084672:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075028.563666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075028.563673:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075028.563676:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075028.563682:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2002 00000100:00080000:3.0:1769075028.563692:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1769075028.563698:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075028.563728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617086464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075028.563792:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617086464:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075028.563915:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075028.563920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075028.563922:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075028.563928:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2021 00000100:00080000:3.0:1769075028.563933:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1769075028.563939:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075028.563949:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075028.563967:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617086592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075028.564009:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086592:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075028.564183:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075028.564189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075028.564192:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075028.564196:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2021 00000100:00080000:3.0:1769075028.564201:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1769075028.564205:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075028.564211:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075028.564226:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617086720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075028.564273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075028.564276:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075028.564278:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075028.564282:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2002 00000100:00080000:3.0:1769075028.564287:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1769075028.564291:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075028.564296:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075028.564307:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617086848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075028.564315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075028.564318:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075028.564319:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075028.564322:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2002 00000100:00080000:3.0:1769075028.564325:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1769075028.564329:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075028.564333:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075028.564344:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617086976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075028.564461:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086720:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075028.564497:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086848:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075028.564521:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086976:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075029.075284:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075029.459366:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075030.099379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075030.483393:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075030.611522:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.204.160:988 00000400:00000100:1.0:1769075030.611563:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075030.611588:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075030.611625:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075030.611637:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075030.611648:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075030.611666:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769075030.611944:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075028/real 1769075030] req@ffff9b0352a19500 x1855007617086720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075030.611969:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617086720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075030.611982:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075030.611987:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075030.611992:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086720:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075030.612000:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075028/real 1769075030] req@ffff9b0352a18380 x1855007617086848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075030.612012:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617086848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075030.612019:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075030.612023:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075030.612026:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086848:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075030.612033:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075028/real 1769075030] req@ffff9b0352a19f80 x1855007617086976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075030.612052:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617086976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075030.612059:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075030.612062:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075030.612065:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086976:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075030.612073:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075028/real 1769075030] req@ffff9b0352a1b100 x1855007617086592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075030.612085:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617086592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075030.612093:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075030.612096:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075030.612099:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617086592:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075030.612106:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075028/real 1769075030] req@ffff9b0352a1ad80 x1855007617086464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075030.612118:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617086464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075030.612125:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075030.612128:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075030.612131:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617086464:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075031.123322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075031.507364:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075032.147331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075032.531490:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769075032.659345:0:2407:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074977/real 1769074977] req@ffff9b0342ce9f80 x1855007617081344/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075032 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:0.0:1769075032.659366:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074977/real 1769074977] req@ffff9b0342ce9c00 x1855007617081216/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075032 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075032.659408:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342ce9c00 x1855007617081216/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075032 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075032.659415:0:2406:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 12, was 9): DISCONN 00000100:00100000:0.0:1769075032.659429:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617081216:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769075032.670775:0:2407:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342ce9f80 x1855007617081344/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075032 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075032.670785:0:2407:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 4, was 1): DISCONN 00000100:00100000:1.0:1769075032.670803:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ce9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617081344:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769075033.171304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075033.555424:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075033.683349:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075033.683365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075033.683372:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075033.683388:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2022 00000100:00080000:3.0:1769075033.683402:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075033.683411:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075033.683458:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617087104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075033.683487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075033.683490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075033.683491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075033.683495:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2022 00000100:00080000:3.0:1769075033.683500:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1769075033.683504:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075033.683513:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075033.683521:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617087104:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075033.683528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617087232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075033.683543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075033.683546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075033.683547:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075033.683552:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2022 00000100:00080000:3.0:1769075033.683556:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1769075033.683560:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075033.683564:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075033.683575:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617087360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075033.683583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075033.683586:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075033.683587:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075033.683591:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2022 00000100:00080000:3.0:1769075033.683593:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075033.683596:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075033.683599:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075033.683612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617087488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075033.683622:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075033.683626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075033.683628:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075033.683633:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617087232:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075033.683635:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2022 00000100:00080000:3.0:1769075033.683642:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075033.683649:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075033.683654:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075033.683666:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617087616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075033.683678:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617087360:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075033.683715:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617087488:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075033.683768:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617087616:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075034.195337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075034.579349:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075035.219323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075035.603353:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075036.243309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075036.627391:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769075036.755512:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.204.160:988 00000400:00000100:3.0:1769075036.755551:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769075036.755565:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075036.755596:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075036.755607:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075036.755617:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075036.755629:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769075036.755637:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075033/real 1769075036] req@ffff9b0352a19c00 x1855007617087232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075036.755677:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617087232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075036.755690:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075036.755711:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075036.755717:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617087232:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075036.755727:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075033/real 1769075036] req@ffff9b0352a1b480 x1855007617087360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075036.755739:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617087360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075036.755747:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075036.755750:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075036.755754:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617087360:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075036.755761:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075033/real 1769075036] req@ffff9b0352a1aa00 x1855007617087488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075036.755774:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617087488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075036.755782:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075036.755786:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075036.755789:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617087488:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075036.755796:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075033/real 1769075036] req@ffff9b0352a18700 x1855007617087104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075036.755811:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617087104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075036.755818:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075036.755821:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075036.755825:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617087104:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075036.755859:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075033/real 1769075036] req@ffff9b0352a18e00 x1855007617087616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075036.755873:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617087616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075036.755880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075036.755883:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075036.755886:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617087616:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075037.267390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075037.651388:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769075037.779368:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074982/real 1769074982] req@ffff9b0365a3aa00 x1855007617081856/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075037 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075037.779411:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0365a3aa00 x1855007617081856/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075037 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075037.779421:0:2406:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 13, was 9): DISCONN 00000100:00000400:0.0:1769075037.779427:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074982/real 1769074982] req@ffff9b0365a3b480 x1855007617081984/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075037 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075037.779453:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0365a3b480 x1855007617081984/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075037 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075037.779461:0:2406:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 5, was 1): DISCONN 00000100:00100000:0.0:1769075037.779475:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617081856:192.168.204.160@tcp:400:kworker.0 00000100:00100000:0.0:1769075037.779483:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617081984:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769075038.291338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075038.675385:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769075038.803347:0:2407:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074983/real 1769074983] req@ffff9b0342ceb800 x1855007617082624/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075038 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075038.803382:0:2407:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342ceb800 x1855007617082624/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075038 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075038.803390:0:2407:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 5, was 1): DISCONN 00000100:00000400:1.0:1769075038.803397:0:2407:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074983/real 1769074983] req@ffff9b0342cead80 x1855007617082496/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075038 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075038.803400:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075038.803409:0:2407:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0342cead80 x1855007617082496/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075038 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075038.803415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075038.803422:0:2407:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 13, was 9): DISCONN 00000100:00080000:3.0:1769075038.803426:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075038.803441:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2027 00000100:00100000:1.0:1769075038.803442:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342ceb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617082624:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769075038.803457:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0342cead80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617082496:192.168.204.160@tcp:400:kworker.0 00000100:00080000:3.0:1769075038.803463:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075038.803478:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075038.803521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617087744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075038.803553:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075038.803558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075038.803560:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075038.803564:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2027 00000100:00080000:3.0:1769075038.803568:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075038.803574:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075038.803586:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075038.803602:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617087872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075038.803617:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617087744:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075038.803621:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075038.803645:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075038.803648:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075038.803666:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2027 00000100:00080000:3.0:1769075038.803673:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075038.803679:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075038.803686:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075038.803701:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617088000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075038.803713:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769075038.803717:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075038.803723:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075038.803726:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075038.803734:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2027 00000100:00080000:3.0:1769075038.803744:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075038.803750:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075038.803758:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075038.803776:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617088128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075038.803787:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075038.803792:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075038.803795:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075038.803802:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075038/real 1769075038] req@ffff9b0352a1a300 x1855007617087744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075038.803804:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2027 00000100:00080000:3.0:1769075038.803812:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075038.803824:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075038.803835:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075038.803852:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617087744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075038.803855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617088256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075038.803879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075038.803892:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075038.803898:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617087744:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075038.803924:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617087872:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075038.803966:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075038.803981:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075038/real 1769075038] req@ffff9b0352a18e00 x1855007617087872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075038.803996:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617087872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075038.804005:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075038.804010:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075038.804020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617087872:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075038.804027:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088000:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075038.804049:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075038.804060:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075038/real 1769075038] req@ffff9b0352a18700 x1855007617088000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075038.804072:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617088000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075038.804080:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075038.804084:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075038.804088:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088000:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075038.804095:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088128:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075038.804130:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075038.804142:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075038/real 1769075038] req@ffff9b0352a1aa00 x1855007617088128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075038.804162:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617088128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075038.804170:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075038.804174:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075038.804178:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088128:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075038.804185:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088256:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075038.804207:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075038.804218:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075038/real 1769075038] req@ffff9b0352a1b480 x1855007617088256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075038.804229:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617088256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075038.804273:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075038.804277:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075038.804281:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088256:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075039.315310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075039.699397:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075040.339340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075040.723413:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075041.363298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075041.747385:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075042.387362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075042.771488:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075043.411316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075043.795360:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075043.923350:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000400:2.0:1769075043.923362:0:2408:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074988/real 1769074988] req@ffff9b03482c7100 x1855007617083392/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075043 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075043.923364:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075043.923370:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075043.923381:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2033 00000100:00080000:3.0:1769075043.923391:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075043.923392:0:2408:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b03482c7100 x1855007617083392/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075043 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075043.923402:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769075043.923403:0:2408:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 6, was 1): DISCONN 00000100:00000400:2.0:1769075043.923412:0:2408:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074988/real 1769074988] req@ffff9b03482c4000 x1855007617083264/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075043 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075043.923421:0:2408:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b03482c4000 x1855007617083264/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075043 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075043.923425:0:2408:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 14, was 9): DISCONN 00000100:00100000:3.0:1769075043.923436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617088384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075043.923437:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617083392:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769075043.923447:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617083264:192.168.204.160@tcp:400:kworker.0 00000100:00080000:3.0:1769075043.923469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075043.923473:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075043.923475:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075043.923480:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2033 00000100:00080000:3.0:1769075043.923484:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075043.923487:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075043.923496:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075043.923500:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617088384:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075043.923508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617088512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075043.923518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075043.923520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075043.923521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075043.923524:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2033 00000100:00080000:3.0:1769075043.923526:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075043.923528:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075043.923531:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075043.923539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617088640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075043.923545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075043.923561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075043.923562:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075043.923566:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2033 00000100:00080000:3.0:1769075043.923569:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075043.923572:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075043.923576:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075043.923587:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617088768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075043.923595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075043.923597:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769075043.923600:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088512:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075043.923600:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075043.923608:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2033 00000100:00080000:3.0:1769075043.923612:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075043.923615:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075043.923620:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075043.923630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617088896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075043.923633:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088640:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075043.923659:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088768:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075043.923677:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088896:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075044.435324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075044.819428:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075045.459351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075045.843412:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075046.483329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075046.867378:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769075046.995673:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.204.160:988 00000400:00000100:3.0:1769075046.995738:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769075046.995760:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075046.995810:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075046.995823:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075046.995837:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075046.995849:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769075046.995862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075043/real 1769075046] req@ffff9b0352a1ad80 x1855007617088512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075046.995896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617088512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075046.995912:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075046.995919:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075046.995924:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088512:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075046.995934:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075043/real 1769075046] req@ffff9b0352a1b100 x1855007617088640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075046.995948:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617088640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075046.995957:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075046.995962:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075046.995966:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088640:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075046.995994:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075043/real 1769075046] req@ffff9b0352a19f80 x1855007617088768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075046.996007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617088768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075046.996015:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075046.996024:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075046.996028:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088768:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075046.996036:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075043/real 1769075046] req@ffff9b0352a19c00 x1855007617088384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075046.996050:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617088384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075046.996058:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075046.996063:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075046.996067:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617088384:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075046.996101:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075043/real 1769075046] req@ffff9b0352a18380 x1855007617088896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075046.996115:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617088896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075046.996123:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075046.996127:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075046.996131:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617088896:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075047.507362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075047.891423:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769075048.019359:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074993/real 1769074993] req@ffff9b0343def480 x1855007617083648/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075048 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075048.019399:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0343def480 x1855007617083648/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075048 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075048.019408:0:2406:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 7, was 1): DISCONN 00000100:00100000:0.0:1769075048.019424:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617083648:192.168.204.160@tcp:400:kworker.0 00000100:00000400:0.0:1769075048.019500:0:2407:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074993/real 1769074993] req@ffff9b0343ded500 x1855007617083520/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075048 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075048.019520:0:2407:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0343ded500 x1855007617083520/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075048 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075048.019527:0:2407:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 15, was 9): DISCONN 00000100:00100000:0.0:1769075048.019535:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617083520:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769075048.531325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075048.915386:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075049.043363:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075049.043379:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075049.043384:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075049.043398:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2038 00000100:00080000:1.0:1769075049.043410:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075049.043419:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075049.043479:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617089024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075049.043513:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075049.043516:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075049.043518:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075049.043522:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2038 00000100:00080000:1.0:1769075049.043525:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075049.043529:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075049.043538:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075049.043550:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617089152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075049.043559:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075049.043562:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075049.043564:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075049.043570:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617089024:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075049.043572:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2038 00000100:00080000:1.0:1769075049.043579:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075049.043586:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075049.043592:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075049.043609:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617089280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075049.043621:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075049.043624:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075049.043626:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075049.043630:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2038 00000100:00080000:1.0:1769075049.043654:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075049.043660:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075049.043667:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769075049.043668:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075049.043685:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617089408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075049.043697:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075049.043700:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075049.043702:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075049.043708:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2038 00000100:00080000:1.0:1769075049.043713:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075049.043719:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075049.043725:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075049.043727:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075049/real 1769075049] req@ffff9b0343dee680 x1855007617089024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075049.043745:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617089536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075049.043757:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617089024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075049.043772:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075049.043778:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075049.043783:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617089024:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075049.043808:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089152:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075049.043835:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075049.043849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075049/real 1769075049] req@ffff9b0343defb80 x1855007617089152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075049.043870:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617089152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075049.043885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075049.043889:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075049.043893:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089152:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075049.043901:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089280:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075049.043923:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075049.043936:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075049/real 1769075049] req@ffff9b0343dece00 x1855007617089280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075049.043947:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617089280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075049.043956:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075049.043968:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075049.043971:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089280:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075049.043977:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089408:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075049.043999:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075049.044010:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075049/real 1769075049] req@ffff9b0343deca80 x1855007617089408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075049.044023:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617089408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075049.044031:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075049.044033:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075049.044037:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089408:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075049.044043:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089536:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075049.044061:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075049.044071:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075049/real 1769075049] req@ffff9b0343deed80 x1855007617089536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075049.044081:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617089536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075049.044090:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075049.044093:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075049.044095:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089536:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075049.555341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075049.939419:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075050.579341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075050.963333:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075051.603337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075051.987385:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075052.627365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075053.011374:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1769075053.139412:0:2408:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074998/real 1769074998] req@ffff9b0352a1b800 x1855007617084160/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075053 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075053.148755:0:2408:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0352a1b800 x1855007617084160/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075053 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075053.148765:0:2408:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 16, was 9): DISCONN 00000100:00100000:2.0:1769075053.148785:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617084160:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769075053.651333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00000400:2.0:1769075053.651451:0:2409:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769074998/real 1769074998] req@ffff9b0352a18000 x1855007617084288/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075053 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075053.651479:0:2409:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0352a18000 x1855007617084288/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075053 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075053.651486:0:2409:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 8, was 1): DISCONN 00000100:00100000:2.0:1769075053.651497:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617084288:192.168.204.160@tcp:400:kworker.0 00010000:00010000:2.0:1769075054.035394:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075054.163359:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075054.163368:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075054.163372:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075054.163391:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2043 00000100:00080000:2.0:1769075054.163400:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075054.163406:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075054.163437:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617089664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075054.163460:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075054.163463:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075054.163464:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075054.163468:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2043 00000100:00080000:2.0:1769075054.163470:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075054.163473:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075054.163484:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075054.163494:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617089792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075054.163507:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075054.163510:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075054.163512:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075054.163515:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617089664:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075054.163518:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2043 00000100:00080000:2.0:1769075054.163525:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075054.163527:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075054.163530:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075054.163540:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617089920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075054.163546:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075054.163549:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075054.163551:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075054.163553:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2043 00000100:00080000:2.0:1769075054.163557:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075054.163559:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075054.163562:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075054.163569:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617090048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075054.163576:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769075054.163579:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075054.163581:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075054.163584:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075054.163588:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2043 00000100:00080000:2.0:1769075054.163591:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075054.163594:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075054.163597:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075054.163606:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617090176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075054.163630:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075054/real 1769075054] req@ffff9b03482c4000 x1855007617089664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075054.163646:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617089664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075054.163657:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075054.163662:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075054.163667:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617089664:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075054.163686:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089792:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075054.163704:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075054.163713:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075054/real 1769075054] req@ffff9b03482c7100 x1855007617089792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075054.163721:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617089792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075054.163727:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075054.163729:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075054.163731:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089792:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075054.163735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089920:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075054.163745:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075054.163751:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075054/real 1769075054] req@ffff9b03482c4700 x1855007617089920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075054.163766:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617089920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075054.163772:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075054.163774:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075054.163776:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617089920:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075054.163780:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090048:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075054.163791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075054.163797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075054/real 1769075054] req@ffff9b03482c5180 x1855007617090048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075054.163809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617090048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075054.163814:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075054.163816:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075054.163818:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090048:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075054.163821:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090176:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075054.163831:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075054.163836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075054/real 1769075054] req@ffff9b03482c6300 x1855007617090176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075054.163845:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617090176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075054.163854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075054.163857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075054.163861:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090176:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075054.675322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075055.059384:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075055.699336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075056.083359:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075056.723319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075057.107440:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075057.747337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075058.131382:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1769075058.259414:0:2408:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075003/real 1769075003] req@ffff9b03482c7800 x1855007617084416/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075058 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075058.259448:0:2408:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b03482c7800 x1855007617084416/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075058 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075058.259458:0:2408:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 17, was 9): DISCONN 00000100:00100000:2.0:1769075058.259486:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617084416:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769075058.771326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00000400:2.0:1769075058.771334:0:2409:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075003/real 1769075003] req@ffff9b03482c5500 x1855007617084544/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075058 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075058.771359:0:2409:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b03482c5500 x1855007617084544/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075058 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075058.771366:0:2409:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 9, was 1): DISCONN 00000100:00100000:2.0:1769075058.771378:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617084544:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769075059.155491:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075059.283338:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075059.283347:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075059.283351:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075059.283360:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2048 00000100:00080000:2.0:1769075059.283369:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075059.283376:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075059.283405:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617090304/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075059.283427:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075059.283430:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075059.283432:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075059.283435:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2048 00000100:00080000:2.0:1769075059.283437:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075059.283440:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075059.283447:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075059.283457:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7800 x1855007617090432/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075059.283466:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769075059.283468:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617090304:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075059.283469:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075059.283473:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075059.283481:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2048 00000100:00080000:2.0:1769075059.283484:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075059.283487:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075059.283490:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075059.283501:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617090560/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075059.283508:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075059.283511:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075059.283513:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075059.283517:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2048 00000100:00080000:2.0:1769075059.283520:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075059.283523:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075059.283528:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075059.283537:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617090688/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075059.283544:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075059.283547:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075059.283548:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075059.283551:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2048 00000100:00080000:2.0:1769075059.283555:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075059.283559:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075059.283562:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075059.283570:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617090816/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075059.283575:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090432:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075059.283611:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090560:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075059.283637:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090688:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075059.283667:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090816:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075059.795350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075060.179466:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075060.819303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075061.203410:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075061.843385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075062.227392:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075062.355558:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1020 -> 192.168.204.160:988 00000400:00000100:1.0:1769075062.355597:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075062.355612:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075062.355642:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075062.355649:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075062.355654:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075062.355660:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769075062.355698:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075059/real 1769075062] req@ffff9b03482c6d80 x1855007617090816/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075062.355726:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617090816/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075062.355736:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075062.355742:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075062.355747:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090816:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075062.355756:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075059/real 1769075062] req@ffff9b03482c7800 x1855007617090432/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075062.355769:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7800 x1855007617090432/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075062.355788:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075062.355792:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075062.355795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090432:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075062.355802:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075059/real 1769075062] req@ffff9b03482c4a80 x1855007617090560/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075062.355818:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617090560/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075062.355835:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075062.355839:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075062.355843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090560:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075062.355851:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075059/real 1769075062] req@ffff9b03482c4380 x1855007617090688/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075062.355863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617090688/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075062.355870:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075062.355874:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075062.355877:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617090688:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075062.355883:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075059/real 1769075062] req@ffff9b03482c5500 x1855007617090304/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075062.355895:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617090304/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075062.355903:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075062.355907:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075062.355910:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617090304:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075062.867389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075063.251409:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1769075063.379344:0:2407:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075008/real 1769075008] req@ffff9b0365a38000 x1855007617084928/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075063 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075063.379378:0:2407:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0365a38000 x1855007617084928/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075063 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075063.379389:0:2407:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 10, was 1): DISCONN 00000100:00000400:1.0:1769075063.379398:0:2407:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075008/real 1769075008] req@ffff9b0365a38a80 x1855007617084800/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075063 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075063.379413:0:2407:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0365a38a80 x1855007617084800/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075063 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075063.379419:0:2407:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 18, was 9): DISCONN 00000100:00100000:1.0:1769075063.379428:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617084928:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769075063.379435:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617084800:192.168.204.160@tcp:400:kworker.0 00000100:00100000:2.0:1769075063.891388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075064.275385:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075064.403518:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075064.403529:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075064.403533:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075064.403545:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2053 00000100:00080000:2.0:1769075064.403555:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075064.403565:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075064.403609:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617090944/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075064.403678:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617090944:192.168.204.160@tcp:250:kworker.0 00000800:00000100:0.0:1769075064.403740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075064.403773:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075064.403778:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075064.403780:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075064.403786:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2053 00000100:00080000:2.0:1769075064.403790:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075064.403795:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075064.403805:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075064.403827:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617091072/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075064.403887:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075064/real 1769075064] req@ffff9b03482c5500 x1855007617090944/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075064.403905:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617090944/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075064.403916:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075064.403921:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075064.403926:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617090944:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075064.403945:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091072:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075064.403968:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075064.403981:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075064/real 1769075064] req@ffff9b03482c4380 x1855007617091072/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075064.403994:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617091072/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075064.404004:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075064.404007:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075064.404011:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091072:192.168.204.160@tcp:38:kworker.0 00000100:00080000:2.0:1769075064.404080:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075064.404085:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075064.404087:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075064.404091:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2053 00000100:00080000:2.0:1769075064.404096:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075064.404101:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075064.404107:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075064.404122:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617091200/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075064.404135:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075064.404138:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075064.404140:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075064.404144:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2053 00000100:00080000:2.0:1769075064.404148:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075064.404152:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075064.404159:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075064.404170:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7800 x1855007617091328/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075064.404179:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075064.404182:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075064.404184:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075064.404188:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2053 00000100:00080000:2.0:1769075064.404191:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075064.404196:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075064.404201:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075064.404228:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617091456/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075064.404463:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091200:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075064.404492:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075064.404506:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075064/real 1769075064] req@ffff9b03482c4a80 x1855007617091200/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075064.404519:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617091200/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075064.404527:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075064.404531:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075064.404535:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091200:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075064.404542:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091328:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075064.404564:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075064.404576:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075064/real 1769075064] req@ffff9b03482c7800 x1855007617091328/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075064.404589:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7800 x1855007617091328/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075064.404597:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075064.404600:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075064.404603:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091328:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075064.404611:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091456:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075064.404630:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075064.404639:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075064/real 1769075064] req@ffff9b03482c6d80 x1855007617091456/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075064.404654:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617091456/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075064.404661:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075064.404665:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075064.404668:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091456:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075064.915403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075065.299373:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075065.939312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075066.323381:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075066.963302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075067.347431:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075067.987310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075068.371368:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1769075068.499353:0:2406:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075013/real 1769075013] req@ffff9b0365a39f80 x1855007617085440/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075068 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075068.499398:0:2406:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0365a39f80 x1855007617085440/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075068 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075068.499408:0:2406:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 11, was 1): DISCONN 00000100:00000400:1.0:1769075068.499413:0:2407:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075013/real 1769075013] req@ffff9b0365a38e00 x1855007617085312/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075068 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075068.499424:0:2406:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2406:1855007617085440:192.168.204.160@tcp:400:kworker.0 00000100:00100000:1.0:1769075068.499439:0:2407:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0365a38e00 x1855007617085312/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075068 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075068.499446:0:2407:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 19, was 9): DISCONN 00000100:00100000:1.0:1769075068.499455:0:2407:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2407:1855007617085312:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769075069.011349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075069.395397:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075069.523346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075069.523358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075069.523363:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075069.523375:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2058 00000100:00080000:3.0:1769075069.523385:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075069.523393:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075069.523452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617091584/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075069.523481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075069.523484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075069.523485:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075069.523489:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2058 00000100:00080000:3.0:1769075069.523491:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075069.523494:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075069.523504:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075069.523515:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617091712/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075069.523524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075069.523527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075069.523528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075069.523532:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2058 00000100:00080000:3.0:1769075069.523534:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075069.523536:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075069.523540:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075069.523547:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617091584:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075069.523551:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617091840/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075069.523568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075069.523587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075069.523589:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075069.523592:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2058 00000100:00080000:3.0:1769075069.523595:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075069.523598:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075069.523602:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075069.523611:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617091968/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075069.523617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075069.523620:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075069.523621:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075069.523624:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2058 00000100:00080000:3.0:1769075069.523627:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075069.523629:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075069.523632:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075069.523640:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617092096/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075069.523656:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075069.523723:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075069/real 1769075069] req@ffff9b0352a19500 x1855007617091584/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075069.523752:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617091584/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075069.523766:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075069.523771:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075069.523777:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617091584:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075069.523786:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091712:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075069.523806:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075069.523817:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075069/real 1769075069] req@ffff9b0352a18000 x1855007617091712/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075069.523826:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617091712/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075069.523833:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075069.523836:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075069.523838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091712:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075069.523859:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091840:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075069.523872:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075069.523878:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075069/real 1769075069] req@ffff9b0352a1b800 x1855007617091840/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075069.523886:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617091840/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075069.523892:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075069.523894:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075069.523896:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091840:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075069.523900:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091968:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075069.523910:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075069.523916:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075069/real 1769075069] req@ffff9b0352a18380 x1855007617091968/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075069.523923:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617091968/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075069.523928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075069.523930:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075069.523932:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617091968:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075069.523935:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092096:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075069.523945:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075069.523950:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075069/real 1769075069] req@ffff9b0352a19c00 x1855007617092096/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075069.523957:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617092096/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075069.523962:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075069.523964:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075069.523966:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092096:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075070.035313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075070.419371:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075071.059364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075071.443420:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075072.083314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075072.467404:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075073.107302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00000400:2.0:1769075073.107313:0:2409:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075018/real 1769075018] req@ffff9b0352a18a80 x1855007617085696/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075073 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075073.107347:0:2409:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0352a18a80 x1855007617085696/t0(0) o400->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075073 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075073.107361:0:2409:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0352ab6000 already not connected (conn 12, was 1): DISCONN 00000100:00100000:2.0:1769075073.107381:0:2409:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2409:1855007617085696:192.168.204.160@tcp:400:kworker.0 00010000:00010000:3.0:1769075073.491405:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1769075073.619357:0:2408:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1769075018/real 1769075018] req@ffff9b0352a1a680 x1855007617085568/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075073 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075073.619408:0:2408:0:(client.c:2522:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9b0352a1a680 x1855007617085568/t0(0) o400->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 224/224 e 0 to 1 dl 1769075073 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075073.619418:0:2408:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff9b0343fe4000 already not connected (conn 20, was 9): DISCONN 00000100:00100000:3.0:1769075073.619454:0:2408:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2408:1855007617085568:192.168.204.160@tcp:400:kworker.0 00000100:00100000:3.0:1769075074.131313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075074.515386:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075074.643336:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075074.643346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075074.643350:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075074.643359:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2063 00000100:00080000:1.0:1769075074.643366:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075074.643372:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075074.643409:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617092224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075074.643432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075074.643437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075074.643439:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075074.643442:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2063 00000100:00080000:1.0:1769075074.643444:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075074.643447:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075074.643456:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075074.643465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617092352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075074.643473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075074.643476:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075074.643477:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075074.643482:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617092224:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075074.643483:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2063 00000100:00080000:1.0:1769075074.643489:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075074.643493:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075074.643497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075074.643505:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedc00 x1855007617092480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075074.643511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075074.643514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075074.643515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075074.643519:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2063 00000100:00080000:1.0:1769075074.643521:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075074.643524:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075074.643527:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075074.643534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617092608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075074.643539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075074.643542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075074.643543:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075074.643545:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2063 00000100:00080000:1.0:1769075074.643558:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075074.643561:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075074.643564:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769075074.643569:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075074.643578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617092736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075074.643606:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075074/real 1769075074] req@ffff9b0343deea00 x1855007617092224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075074.643621:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617092224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075074.643629:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075074.643634:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075074.643638:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617092224:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075074.643662:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092352:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075074.643688:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075074.643701:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075074/real 1769075074] req@ffff9b0343def800 x1855007617092352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075074.643718:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617092352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075074.643724:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075074.643727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075074.643730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092352:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075074.643736:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092480:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075074.643767:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075074.643779:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075074/real 1769075074] req@ffff9b0343dedc00 x1855007617092480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075074.643792:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedc00 x1855007617092480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075074.643813:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075074.643816:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075074.643818:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092480:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075074.643823:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092608:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075074.643838:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075074.643846:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075074/real 1769075074] req@ffff9b0343ded880 x1855007617092608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075074.643854:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617092608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075074.643863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075074.643866:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075074.643868:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092608:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075074.643871:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092736:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075074.643881:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075074.643886:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075074/real 1769075074] req@ffff9b0343def100 x1855007617092736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075074.643900:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617092736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075074.643905:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075074.643907:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075074.643909:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092736:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075075.155396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075075.539378:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075076.179387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075076.563483:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075077.203440:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075077.587391:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075078.227357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075078.611434:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075079.251401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075079.635362:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075079.763377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075079.763389:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075079.763394:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075079.763405:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2068 00000100:00080000:3.0:1769075079.763416:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075079.763423:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075079.763465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617092864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075079.763489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075079.763492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075079.763493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075079.763496:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2068 00000100:00080000:3.0:1769075079.763499:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075079.763501:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075079.763511:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075079.763520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617092992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075079.763528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075079.763531:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075079.763532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075079.763536:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2068 00000100:00080000:3.0:1769075079.763539:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075079.763542:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617092864:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075079.763544:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075079.763551:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075079.763559:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617093120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075079.763566:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075079.763569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075079.763570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075079.763573:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2068 00000100:00080000:3.0:1769075079.763575:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075079.763578:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075079.763581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075079.763590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617093248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075079.763596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075079.763598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075079.763600:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075079.763603:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2068 00000100:00080000:3.0:1769075079.763605:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075079.763608:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075079.763611:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075079.763618:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617093376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075079.763665:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092992:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075079.763702:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093120:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075079.763720:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093248:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075079.763734:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093376:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075080.275289:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075080.659467:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075081.299371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075081.683381:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075082.323346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075082.707379:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075082.835469:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769075082.835514:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075082.835528:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075082.835571:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075082.835578:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075082.835583:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075082.835588:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769075082.835641:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075079/real 1769075082] req@ffff9b0352a1b100 x1855007617093120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075082.835663:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617093120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075082.835676:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075082.835681:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075082.835686:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093120:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075082.835694:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075079/real 1769075082] req@ffff9b0352a1ad80 x1855007617093248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075082.835708:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617093248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075082.835716:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075082.835720:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075082.835724:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093248:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075082.835731:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075079/real 1769075082] req@ffff9b0352a1b480 x1855007617093376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075082.835743:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617093376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075082.835751:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075082.835755:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075082.835759:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093376:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075082.835765:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075079/real 1769075082] req@ffff9b0352a1a680 x1855007617092864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075082.835777:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617092864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075082.835785:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075082.835789:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075082.835792:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617092864:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075082.835800:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075079/real 1769075082] req@ffff9b0352a19f80 x1855007617092992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075082.835813:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617092992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075082.835821:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075082.835826:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075082.835829:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617092992:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075083.347306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075083.731393:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075084.371285:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075084.755405:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075084.883327:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075084.883336:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075084.883340:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075084.883348:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2074 00000100:00080000:2.0:1769075084.883356:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075084.883364:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075084.883400:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617093504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075084.883428:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075084.883431:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075084.883432:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075084.883436:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2074 00000100:00080000:2.0:1769075084.883439:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075084.883450:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075084.883458:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075084.883470:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617093632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075084.883478:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617093504:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075084.883481:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075084.883485:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075084.883487:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075084.883490:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2074 00000100:00080000:2.0:1769075084.883492:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075084.883498:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075084.883502:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075084.883511:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617093760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075084.883518:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075084.883522:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075084.883524:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075084.883528:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2074 00000100:00080000:2.0:1769075084.883532:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075084.883535:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075084.883540:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075084.883551:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617093888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075084.883560:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075084.883563:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075084.883565:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769075084.883570:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075084.883575:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2074 00000100:00080000:2.0:1769075084.883583:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075084.883588:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075084.883594:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075084.883607:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617094016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075084.883618:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075084/real 1769075084] req@ffff9b03482c6300 x1855007617093504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075084.883640:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617093504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075084.883650:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075084.883654:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075084.883659:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617093504:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075084.883685:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093632:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075084.883712:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075084.883726:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075084/real 1769075084] req@ffff9b03482c5180 x1855007617093632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075084.883740:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617093632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075084.883748:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075084.883752:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075084.883756:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093632:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075084.883762:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093760:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075084.883785:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075084.883796:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075084/real 1769075084] req@ffff9b03482c4700 x1855007617093760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075084.883810:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617093760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075084.883818:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075084.883822:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075084.883825:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093760:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075084.883832:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093888:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075084.883854:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075084.883885:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075084/real 1769075084] req@ffff9b03482c7100 x1855007617093888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075084.883900:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617093888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075084.883909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075084.883912:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075084.883916:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617093888:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075084.883922:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094016:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075084.883944:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075084.883956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075084/real 1769075084] req@ffff9b03482c4000 x1855007617094016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075084.883968:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617094016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075084.883977:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075084.883980:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075084.883984:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094016:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075085.395299:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075085.779355:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075086.419330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075086.803369:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075087.443286:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075087.827386:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075088.467316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075088.851461:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075089.491383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075089.875408:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075090.003351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075090.003360:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075090.003364:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075090.003372:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2079 00000100:00080000:3.0:1769075090.003380:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075090.003387:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075090.003418:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617094144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075090.003440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075090.003443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075090.003444:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075090.003446:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2079 00000100:00080000:3.0:1769075090.003449:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075090.003451:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075090.003457:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075090.003465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617094272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075090.003472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075090.003475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075090.003477:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075090.003479:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2079 00000100:00080000:3.0:1769075090.003481:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075090.003484:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075090.003487:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075090.003493:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617094144:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075090.003495:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617094400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075090.003508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075090.003510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075090.003512:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075090.003514:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2079 00000100:00080000:3.0:1769075090.003517:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075090.003520:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075090.003523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075090.003531:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617094528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075090.003537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075090.003539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075090.003540:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075090.003542:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2079 00000100:00080000:3.0:1769075090.003544:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075090.003546:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075090.003550:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075090.003560:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617094656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075090.003561:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075090.003613:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff9b0352a1aa00 x1855007617094144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075090.003628:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617094144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075090.003639:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075090.003644:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075090.003648:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617094144:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075090.003656:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094272:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075090.003678:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075090.003692:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff9b0352a18700 x1855007617094272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075090.003705:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617094272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075090.003711:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075090.003714:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075090.003716:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094272:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075090.003735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094400:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075090.003750:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075090.003757:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff9b0352a18e00 x1855007617094400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075090.003764:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617094400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075090.003769:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075090.003772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075090.003790:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094400:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075090.003795:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094528:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075090.003811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075090.003818:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff9b0352a1a300 x1855007617094528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075090.003826:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617094528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075090.003832:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075090.003833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075090.003836:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094528:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075090.003839:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094656:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075090.003848:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075090.003853:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075090/real 1769075090] req@ffff9b0352a19f80 x1855007617094656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075090.003860:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617094656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075090.003865:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075090.003867:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075090.003869:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094656:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075090.515371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075090.899431:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075091.539356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075091.923451:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075092.563343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075092.947431:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075093.587329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075093.971344:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075094.611379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075094.995403:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075095.123319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075095.123330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075095.123334:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075095.123345:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2084 00000100:00080000:3.0:1769075095.123357:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075095.123367:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075095.123404:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617094784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075095.123432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075095.123436:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075095.123438:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075095.123445:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2084 00000100:00080000:3.0:1769075095.123451:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075095.123456:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075095.123466:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075095.123477:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617094784:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075095.123486:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617094912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075095.123502:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075095.123505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075095.123508:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075095.123513:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2084 00000100:00080000:3.0:1769075095.123529:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075095.123534:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075095.123541:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769075095.123549:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075095.123560:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617095040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075095.123577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075095.123581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075095.123587:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075095.123595:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2084 00000100:00080000:3.0:1769075095.123601:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075095.123607:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075095/real 1769075095] req@ffff9b0352a1a680 x1855007617094784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075095.123610:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075095.123623:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075095.123639:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617094784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075095.123649:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075095.123654:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075095.123658:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617094784:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075095.123677:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094912:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075095.123713:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075095.123729:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617095168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075095.123730:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075095/real 1769075095] req@ffff9b0352a1b480 x1855007617094912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075095.123756:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617094912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075095.123761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075095.123764:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075095.123766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075095.123769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769075095.123772:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075095.123774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617094912:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075095.123785:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095040:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075095.123785:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2084 00000100:00080000:3.0:1769075095.123795:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075095.123803:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769075095.123806:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075095.123816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075095/real 1769075095] req@ffff9b0352a1ad80 x1855007617095040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075095.123832:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617095040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075095.123842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075095.123845:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075095.123847:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095040:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075095.123858:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095168:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075095.123870:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075095.123877:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075095/real 1769075095] req@ffff9b0352a1b100 x1855007617095168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075095.123885:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617095168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075095.123890:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075095.123893:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075095.123895:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095168:192.168.204.160@tcp:8:kworker.0 00000008:00100000:3.0:1769075095.123957:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075095.123975:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617095296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075095.124089:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095296:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075095.124118:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075095.124130:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075095/real 1769075095] req@ffff9b0352a18a80 x1855007617095296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075095.124141:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617095296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075095.124147:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075095.124149:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075095.124152:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095296:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075095.635306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075096.019464:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075096.659382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075097.043415:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075097.683316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075098.067378:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075098.707325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075099.091359:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075099.731299:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075100.115429:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075100.243336:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075100.243362:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075100.243366:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075100.243374:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2089 00000100:00080000:3.0:1769075100.243382:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075100.243390:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075100.243430:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617095424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075100.243456:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075100.243460:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075100.243461:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075100.243465:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2089 00000100:00080000:3.0:1769075100.243468:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075100.243472:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075100.243498:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075100.243507:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617095424:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075100.243512:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617095552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075100.243523:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075100.243526:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075100.243527:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075100.243531:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2089 00000100:00080000:3.0:1769075100.243534:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075100.243537:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075100.243540:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075100.243549:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617095680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075100.243567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075100.243570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075100.243571:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075100.243576:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2089 00000800:00000100:1.0:1769075100.243582:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075100.243583:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075100.243590:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075100.243595:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075100.243623:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617095808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075100.243631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075100.243650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075100.243652:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075100.243658:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2089 00000100:00100000:1.0:1769075100.243660:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075100/real 1769075100] req@ffff9b0352a19c00 x1855007617095424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075100.243663:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075100.243672:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075100.243682:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075100.243691:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617095424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075100.243699:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617095936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075100.243745:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075100.243751:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075100.243756:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617095424:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075100.243779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095552:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075100.243801:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075100.243813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075100/real 1769075100] req@ffff9b0352a18380 x1855007617095552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075100.243826:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617095552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075100.243836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075100.243839:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075100.243843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095552:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075100.243850:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095680:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075100.243871:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075100.243883:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075100/real 1769075100] req@ffff9b0352a1b800 x1855007617095680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075100.243896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617095680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075100.243905:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075100.243909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075100.243912:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095680:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075100.243917:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095808:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075100.243935:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075100.243945:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075100/real 1769075100] req@ffff9b0352a18000 x1855007617095808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075100.243974:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617095808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075100.243982:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075100.243985:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075100.243989:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095808:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075100.243996:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095936:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075100.244018:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075100.244030:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075100/real 1769075100] req@ffff9b0352a19500 x1855007617095936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075100.244043:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617095936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075100.244052:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075100.244055:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075100.244059:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617095936:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075100.755330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075101.139327:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075101.779352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075102.163425:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075102.803337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075103.187401:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075103.827415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075104.211409:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075104.851442:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075105.235380:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075105.363310:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075105.363320:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075105.363324:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075105.363332:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2094 00000100:00080000:0.0:1769075105.363339:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075105.363345:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075105.363377:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39f80 x1855007617096064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075105.363407:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075105.363412:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075105.363414:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075105.363419:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2094 00000100:00080000:0.0:1769075105.363424:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075105.363429:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075105.363440:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075105.363444:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617096064:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075105.363455:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617096192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075105.363465:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075105.363468:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075105.363469:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075105.363472:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2094 00000100:00080000:0.0:1769075105.363475:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075105.363478:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075105.363481:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075105.363489:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617096320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075105.363495:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075105.363498:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075105.363499:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075105.363503:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2094 00000800:00000100:1.0:1769075105.363505:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075105.363507:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075105.363511:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075105.363528:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075105.363538:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b800 x1855007617096448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075105.363545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075105.363548:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075105.363549:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075105.363553:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2094 00000100:00080000:0.0:1769075105.363556:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075105.363560:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075105.363563:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075105.363566:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075105/real 1769075105] req@ffff9b0365a39f80 x1855007617096064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075105.363573:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617096576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075105.363595:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39f80 x1855007617096064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075105.363606:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075105.363609:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075105.363612:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617096064:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075105.363632:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096192:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075105.363649:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075105.363658:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075105/real 1769075105] req@ffff9b0365a3b480 x1855007617096192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075105.363667:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617096192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075105.363672:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075105.363674:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075105.363676:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096192:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075105.363680:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096320:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075105.363692:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075105.363702:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075105/real 1769075105] req@ffff9b0365a3aa00 x1855007617096320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075105.363714:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617096320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075105.363729:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075105.363732:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075105.363735:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096320:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075105.363742:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096448:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075105.363759:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075105.363767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075105/real 1769075105] req@ffff9b0365a3b800 x1855007617096448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075105.363775:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b800 x1855007617096448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075105.363780:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075105.363782:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075105.363784:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096448:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075105.363787:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096576:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075105.363796:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075105.363801:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075105/real 1769075105] req@ffff9b0365a39c00 x1855007617096576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075105.363808:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617096576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075105.363813:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075105.363815:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075105.363817:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096576:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075105.875464:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075106.259356:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075106.899306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075107.283512:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075107.923391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075108.307403:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075108.947322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075109.331371:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075109.971289:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075110.355386:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075110.483312:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075110.483322:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075110.483326:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075110.483334:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2099 00000100:00080000:0.0:1769075110.483344:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075110.483350:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075110.483382:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617096704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075110.483403:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075110.483406:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075110.483411:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075110.483416:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2099 00000100:00080000:0.0:1769075110.483418:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075110.483421:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075110.483440:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075110.483451:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617096832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075110.483459:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075110.483462:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075110.483463:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075110.483468:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2099 00000100:00080000:0.0:1769075110.483471:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075110.483471:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617096704:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075110.483477:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075110.483484:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075110.483491:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617096960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075110.483497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075110.483500:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075110.483501:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075110.483504:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2099 00000100:00080000:0.0:1769075110.483506:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075110.483509:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075110.483512:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075110.483520:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617097088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075110.483527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075110.483529:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075110.483531:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075110.483533:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2099 00000100:00080000:0.0:1769075110.483535:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075110.483537:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075110.483540:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075110.483545:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075110.483549:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617097216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075110.483585:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075110/real 1769075110] req@ffff9b0365a39500 x1855007617096704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075110.483600:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617096704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075110.483608:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075110.483612:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075110.483616:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617096704:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075110.483637:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096832:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075110.483654:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075110.483662:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075110/real 1769075110] req@ffff9b0365a3bb80 x1855007617096832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075110.483670:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617096832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075110.483676:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075110.483678:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075110.483680:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096832:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075110.483683:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096960:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075110.483696:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075110.483701:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075110/real 1769075110] req@ffff9b0365a3b100 x1855007617096960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075110.483709:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617096960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075110.483713:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075110.483715:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075110.483717:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617096960:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075110.483721:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097088:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075110.483730:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075110.483735:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075110/real 1769075110] req@ffff9b0365a3ad80 x1855007617097088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075110.483742:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617097088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075110.483747:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075110.483749:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075110.483751:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097088:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075110.483754:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097216:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075110.483762:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075110.483767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075110/real 1769075110] req@ffff9b0365a39180 x1855007617097216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075110.483779:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617097216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075110.483784:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075110.483785:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075110.483787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097216:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075110.995330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075111.379513:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075112.019337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075112.403446:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075113.043314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075113.427473:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075114.067342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075114.451413:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075115.091345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075115.475431:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075115.603333:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075115.603342:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075115.603358:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075115.603366:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2104 00000100:00080000:1.0:1769075115.603385:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075115.603390:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075115.603422:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617097344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075115.603445:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075115.603448:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075115.603449:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075115.603453:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2104 00000100:00080000:1.0:1769075115.603456:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075115.603459:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075115.603466:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075115.603478:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617097344:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075115.603485:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617097472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075115.603494:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075115.603497:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075115.603498:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075115.603501:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2104 00000100:00080000:1.0:1769075115.603503:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075115.603506:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075115.603509:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075115.603517:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617097600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075115.603524:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075115.603526:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075115.603528:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075115.603530:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2104 00000100:00080000:1.0:1769075115.603532:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075115.603535:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075115.603538:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075115.603545:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617097728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075115.603551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075115.603553:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075115.603554:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075115.603557:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2104 00000100:00080000:1.0:1769075115.603559:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075115.603562:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075115.603567:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075115.603575:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617097856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075115.603578:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097472:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075115.603613:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097600:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075115.603629:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097728:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075115.603643:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097856:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075116.115436:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075116.499464:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075117.139391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075117.523465:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075118.163380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075118.547415:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769075118.675507:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:3.0:1769075118.675563:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769075118.675580:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075118.675616:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075118.675624:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075118.675632:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075118.675641:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769075118.675684:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075115/real 1769075118] req@ffff9b0343dee300 x1855007617097856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075118.675714:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617097856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075118.675725:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075118.675732:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075118.675737:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097856:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075118.675745:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075115/real 1769075118] req@ffff9b0343dec380 x1855007617097472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075118.675760:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617097472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075118.675770:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075118.675774:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075118.675778:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097472:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075118.675788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075115/real 1769075118] req@ffff9b0343ded180 x1855007617097600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075118.675800:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617097600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075118.675806:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075118.675810:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075118.675813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097600:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075118.675819:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075115/real 1769075118] req@ffff9b0343dec000 x1855007617097728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075118.675831:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617097728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075118.675839:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075118.675843:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075118.675847:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617097728:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075118.675854:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075115/real 1769075118] req@ffff9b0343dec700 x1855007617097344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075118.675868:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617097344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075118.675876:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075118.675880:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075118.675884:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617097344:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075119.187337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075119.571456:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075120.211318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075120.595427:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075120.723357:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075120.723369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075120.723373:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075120.723383:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2109 00000100:00080000:1.0:1769075120.723394:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075120.723402:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075120.723438:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617097984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075120.723467:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075120.723471:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075120.723473:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075120.723477:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2109 00000100:00080000:1.0:1769075120.723480:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075120.723483:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075120.723492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075120.723506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617098112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075120.723516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075120.723521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075120.723522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075120.723526:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2109 00000100:00080000:1.0:1769075120.723528:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075120.723534:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075120.723539:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617097984:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769075120.723540:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075120.723552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617098240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075120.723561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075120.723564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075120.723566:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075120.723571:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2109 00000100:00080000:1.0:1769075120.723576:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075120.723581:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075120.723588:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075120.723604:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617098368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075120.723617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075120.723622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075120.723624:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075120.723631:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2109 00000800:00000100:0.0:1769075120.723637:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075120.723640:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075120.723650:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075120.723663:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075120.723685:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617098496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075120.723718:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075120/real 1769075120] req@ffff9b0343dedf80 x1855007617097984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075120.723748:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617097984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075120.723764:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075120.723770:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075120.723776:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617097984:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075120.723787:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098112:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075120.723824:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075120.723865:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075120/real 1769075120] req@ffff9b0343dec700 x1855007617098112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075120.723894:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617098112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075120.723906:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075120.723910:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075120.723915:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098112:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075120.723947:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098240:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075120.723977:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075120.723993:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075120/real 1769075120] req@ffff9b0343dec000 x1855007617098240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075120.724010:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617098240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075120.724020:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075120.724025:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075120.724029:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098240:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075120.724038:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098368:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075120.724064:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075120.724079:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075120/real 1769075120] req@ffff9b0343ded180 x1855007617098368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075120.724095:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617098368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075120.724113:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075120.724118:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075120.724123:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098368:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075120.724130:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098496:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075120.724155:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075120.724171:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075120/real 1769075120] req@ffff9b0343dec380 x1855007617098496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075120.724187:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617098496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075120.724198:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075120.724202:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075120.724207:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098496:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075121.235307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075121.619466:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075122.259322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075122.643365:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075123.283342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075123.667430:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075124.307334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075124.691352:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075125.331368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075125.715584:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075125.843337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075125.843347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075125.843350:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075125.843358:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2115 00000100:00080000:3.0:1769075125.843367:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075125.843372:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075125.843403:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617098624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075125.843425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075125.843428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075125.843429:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075125.843432:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2115 00000100:00080000:3.0:1769075125.843434:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075125.843437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075125.843444:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075125.843453:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617098752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075125.843463:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075125.843466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075125.843467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075125.843473:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2115 00000100:00100000:0.0:1769075125.843475:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617098624:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075125.843475:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075125.843479:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075125.843485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075125.843492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617098880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075125.843499:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075125.843501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075125.843503:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075125.843505:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2115 00000100:00080000:3.0:1769075125.843507:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075125.843509:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075125.843513:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075125.843520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617099008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075125.843526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075125.843527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075125.843529:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075125.843532:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2115 00000100:00080000:3.0:1769075125.843535:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075125.843537:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075125.843540:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769075125.843545:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075125.843552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617099136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075125.843597:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075125/real 1769075125] req@ffff9b0352a19500 x1855007617098624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075125.843613:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617098624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075125.843621:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075125.843625:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075125.843628:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617098624:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075125.843647:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098752:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075125.843664:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075125.843672:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075125/real 1769075125] req@ffff9b0352a18000 x1855007617098752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075125.843681:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617098752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075125.843686:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075125.843704:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075125.843707:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098752:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075125.843712:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098880:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075125.843726:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075125.843733:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075125/real 1769075125] req@ffff9b0352a1b800 x1855007617098880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075125.843742:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617098880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075125.843748:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075125.843750:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075125.843753:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617098880:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075125.843757:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099008:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075125.843768:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075125.843775:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075125/real 1769075125] req@ffff9b0352a18380 x1855007617099008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075125.843784:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617099008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075125.843791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075125.843793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075125.843795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099008:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075125.843799:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099136:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075125.843809:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075125.843816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075125/real 1769075125] req@ffff9b0352a19c00 x1855007617099136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075125.843824:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617099136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075125.843845:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075125.843847:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075125.843849:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099136:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075126.355415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075126.739410:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075127.379408:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075127.763537:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075128.403340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075128.787410:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075129.427441:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075129.811426:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075130.451397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075130.835455:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075130.963369:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075130.963382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075130.963387:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075130.963399:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2120 00000100:00080000:2.0:1769075130.963410:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075130.963433:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075130.963476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617099264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075130.963510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075130.963514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075130.963517:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075130.963521:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2120 00000100:00080000:2.0:1769075130.963524:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075130.963529:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075130.963540:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075130.963570:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617099392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075130.963578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075130.963581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075130.963583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075130.963586:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2120 00000100:00080000:2.0:1769075130.963589:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075130.963592:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075130.963596:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075130.963605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617099520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075130.963614:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617099264:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075130.963617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075130.963623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075130.963624:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075130.963628:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2120 00000100:00080000:2.0:1769075130.963630:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075130.963634:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075130.963640:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075130.963654:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617099648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075130.963665:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075130.963670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075130.963672:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075130.963676:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2120 00000100:00080000:2.0:1769075130.963753:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075130.963757:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075130.963762:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075130.963773:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617099776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075130.963778:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075130.963946:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075130/real 1769075130] req@ffff9b03482c6a00 x1855007617099264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075130.964005:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617099264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075130.964016:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075130.964022:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075130.964027:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617099264:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075130.964034:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099392:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075130.964064:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075130.964074:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075130/real 1769075130] req@ffff9b03482c5f80 x1855007617099392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075130.964083:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617099392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075130.964117:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075130.964120:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075130.964123:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099392:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075130.964147:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099520:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075130.964162:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075130.964170:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075130/real 1769075130] req@ffff9b03482c4e00 x1855007617099520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075130.964178:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617099520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075130.964184:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075130.964186:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075130.964188:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099520:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075130.964192:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099648:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075130.964202:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075130.964208:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075130/real 1769075130] req@ffff9b03482c7b80 x1855007617099648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075130.964215:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617099648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075130.964220:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075130.964221:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075130.964224:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099648:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075130.964228:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099776:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075130.964265:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075130.964273:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075130/real 1769075130] req@ffff9b03482c5c00 x1855007617099776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075130.964282:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617099776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075130.964289:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075130.964292:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075130.964295:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617099776:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075131.475406:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075131.859430:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075132.499371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075132.883495:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075133.523330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075133.907480:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075134.547363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075134.931490:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075135.571337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075135.955471:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075136.083371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075136.083380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075136.083384:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075136.083392:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2125 00000100:00080000:1.0:1769075136.083399:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075136.083405:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075136.083449:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617099904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075136.083472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075136.083475:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075136.083477:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075136.083480:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2125 00000100:00080000:1.0:1769075136.083482:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075136.083485:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075136.083493:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075136.083504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617100032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075136.083507:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617099904:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075136.083520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075136.083523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075136.083526:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075136.083529:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2125 00000100:00080000:1.0:1769075136.083531:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075136.083534:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075136.083537:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075136.083546:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617100160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075136.083552:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075136.083554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075136.083555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075136.083558:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2125 00000100:00080000:1.0:1769075136.083562:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075136.083564:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075136.083568:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075136.083572:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075136.083578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedc00 x1855007617100288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075136.083586:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075136.083588:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075136.083590:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075136.083593:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2125 00000100:00080000:1.0:1769075136.083595:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075136.083596:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075136.083599:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075136.083607:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617100416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075136.083617:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075136/real 1769075136] req@ffff9b0343dee300 x1855007617099904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075136.083633:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617099904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075136.083641:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075136.083644:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075136.083647:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617099904:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075136.083668:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100032:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075136.083683:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075136.083692:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075136/real 1769075136] req@ffff9b0343def100 x1855007617100032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075136.083700:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617100032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075136.083706:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075136.083708:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075136.083710:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100032:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075136.083714:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100160:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075136.083724:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075136.083730:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075136/real 1769075136] req@ffff9b0343ded880 x1855007617100160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075136.083737:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617100160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075136.083742:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075136.083744:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075136.083746:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100160:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075136.083750:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100288:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075136.083759:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075136.083765:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075136/real 1769075136] req@ffff9b0343dedc00 x1855007617100288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075136.083773:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedc00 x1855007617100288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075136.083778:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075136.083779:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075136.083781:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100288:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075136.083785:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100416:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075136.083793:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075136.083798:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075136/real 1769075136] req@ffff9b0343def800 x1855007617100416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075136.083805:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617100416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075136.083810:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075136.083812:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075136.083813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100416:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075136.595308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075136.979329:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075137.619364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075138.003411:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075138.643307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075139.027419:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075139.667437:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075140.051907:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075140.691392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075141.075602:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075141.203369:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075141.203380:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075141.203384:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075141.203394:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2130 00000100:00080000:1.0:1769075141.203405:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075141.203412:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075141.203449:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617100544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075141.203484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075141.203490:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075141.203492:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075141.203498:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2130 00000100:00080000:1.0:1769075141.203503:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075141.203510:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075141.203538:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075141.203557:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617100544:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075141.203559:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617100672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075141.203576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075141.203580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075141.203582:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075141.203586:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2130 00000100:00080000:1.0:1769075141.203590:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075141.203595:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075141.203600:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075141.203610:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617100800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075141.203631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075141.203635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075141.203640:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769075141.203641:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075141.203651:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2130 00000100:00080000:1.0:1769075141.203659:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075141.203665:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075141.203673:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075141.203685:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075141/real 1769075141] req@ffff9b0343deea00 x1855007617100544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075141.203693:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617100928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075141.203709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075141.203712:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617100544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075141.203714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075141.203719:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075141.203728:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2130 00000100:00080000:1.0:1769075141.203743:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075141.203746:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075141.203749:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:3.0:1769075141.203756:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000008:00100000:1.0:1769075141.203758:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075141.203763:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617100544:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075141.203774:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617101056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075141.203791:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100672:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075141.203824:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075141.203835:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075141/real 1769075141] req@ffff9b0343deed80 x1855007617100672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075141.203846:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617100672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075141.203853:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075141.203856:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075141.203858:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100672:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075141.203863:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100800:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075141.203875:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075141.203882:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075141/real 1769075141] req@ffff9b0343deca80 x1855007617100800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075141.203891:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617100800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075141.203898:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075141.203901:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075141.203903:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100800:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075141.203908:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100928:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075141.203975:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075141.203991:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075141/real 1769075141] req@ffff9b0343dece00 x1855007617100928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075141.204007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617100928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075141.204017:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075141.204032:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075141.204035:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617100928:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075141.204040:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101056:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075141.204059:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075141.204068:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075141/real 1769075141] req@ffff9b0343defb80 x1855007617101056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075141.204078:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617101056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075141.204084:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075141.204087:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075141.204099:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101056:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075141.715390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075142.099383:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075142.739332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075143.123441:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075143.763315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075144.147418:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075144.787349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075145.171486:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075145.811374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075146.195376:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075146.323398:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075146.323420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075146.323428:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075146.323444:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2135 00000100:00080000:2.0:1769075146.323457:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075146.323466:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075146.323513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617101184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075146.323552:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075146.323557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075146.323560:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075146.323566:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2135 00000100:00080000:2.0:1769075146.323570:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075146.323575:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075146.323587:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075146.323603:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617101312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075146.323618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075146.323626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1769075146.323626:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617101184:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075146.323630:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075146.323642:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2135 00000100:00080000:2.0:1769075146.323663:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075146.323668:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075146.323674:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075146.323693:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617101440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075146.323706:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075146.323710:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075146.323713:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075146.323718:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2135 00000100:00080000:2.0:1769075146.323722:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075146.323726:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075146.323732:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075146.323744:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617101568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075146.323756:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075146.323761:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075146.323763:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075146.323767:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2135 00000100:00080000:2.0:1769075146.323772:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075146.323783:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769075146.323783:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769075146.323796:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075146.323812:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617101696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075146.323875:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075146/real 1769075146] req@ffff9b03482c5880 x1855007617101184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075146.323905:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617101184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075146.323920:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075146.323926:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075146.323932:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617101184:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075146.323971:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101312:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075146.324018:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075146.324044:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075146/real 1769075146] req@ffff9b03482c6680 x1855007617101312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075146.324070:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617101312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075146.324081:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075146.324086:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075146.324090:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101312:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075146.324099:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101440:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075146.324127:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075146.324139:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075146/real 1769075146] req@ffff9b03482c5c00 x1855007617101440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075146.324168:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617101440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075146.324185:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075146.324191:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075146.324197:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101440:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075146.324210:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101568:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075146.324290:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075146.324308:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075146/real 1769075146] req@ffff9b03482c7b80 x1855007617101568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075146.324325:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617101568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075146.324335:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075146.324340:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075146.324344:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101568:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075146.324352:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101696:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075146.324373:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075146.324384:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075146/real 1769075146] req@ffff9b03482c4e00 x1855007617101696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075146.324398:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617101696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075146.324416:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075146.324420:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075146.324424:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101696:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075146.835349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075147.219681:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075147.859330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075148.243758:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075148.883466:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075149.267666:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075149.907324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075150.291685:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075150.931364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075151.315401:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075151.443368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075151.443383:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075151.443389:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075151.443402:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2140 00000100:00080000:2.0:1769075151.443416:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075151.443427:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075151.443479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617101824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075151.443513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075151.443518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075151.443520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075151.443526:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2140 00000100:00080000:2.0:1769075151.443531:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075151.443536:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075151.443546:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075151.443553:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617101824:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075151.443566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617101952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075151.443583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075151.443587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075151.443589:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075151.443595:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2140 00000100:00080000:2.0:1769075151.443600:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075151.443605:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075151.443610:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075151.443645:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617102080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075151.443647:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075151.443664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075151.443669:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075151.443672:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075151.443679:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2140 00000100:00080000:2.0:1769075151.443686:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075151.443690:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075151.443697:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075151.443717:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617102208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075151.443727:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075151/real 1769075151] req@ffff9b03482c5f80 x1855007617101824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075151.443735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075151.443775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075151.443778:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075151.443788:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2140 00000100:00080000:2.0:1769075151.443795:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075151.443801:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075151.443804:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617101824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769075151.443809:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1769075151.443825:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769075151.443831:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617102336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075151.443836:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075151.443850:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617101824:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075151.443889:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101952:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075151.443916:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075151.443930:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075151/real 1769075151] req@ffff9b03482c6a00 x1855007617101952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075151.443944:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617101952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075151.443958:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075151.443962:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075151.443966:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617101952:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075151.443973:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102080:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075151.443991:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075151.444001:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075151/real 1769075151] req@ffff9b03482c4000 x1855007617102080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075151.444014:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617102080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075151.444022:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075151.444025:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075151.444029:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102080:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075151.444035:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102208:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075151.444051:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075151.444060:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075151/real 1769075151] req@ffff9b03482c7100 x1855007617102208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075151.444070:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617102208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075151.444076:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075151.444079:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075151.444082:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102208:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075151.444088:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102336:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075151.444102:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075151.444109:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075151/real 1769075151] req@ffff9b03482c4700 x1855007617102336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075151.444120:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617102336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075151.444127:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075151.444130:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075151.444133:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102336:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075151.955330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075152.339364:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075152.979393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075153.363501:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075154.003355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075154.387477:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075155.027418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075155.411422:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075156.051393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075156.435632:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075156.563456:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075156.563490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075156.563500:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075156.563520:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2145 00000100:00080000:1.0:1769075156.563539:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075156.563553:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075156.563616:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617102464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075156.563717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075156.563723:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075156.563726:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075156.563733:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2145 00000100:00080000:1.0:1769075156.563738:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075156.563762:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075156.563777:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075156.563799:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617102592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075156.563816:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075156.563821:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075156.563825:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075156.563831:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2145 00000100:00080000:1.0:1769075156.563836:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075156.563841:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075156.563849:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075156.563859:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617102464:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075156.563866:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617102720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075156.563891:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075156.563895:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075156.563898:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075156.563903:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2145 00000100:00080000:1.0:1769075156.563910:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075156.563915:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075156.563922:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075156.563941:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617102848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075156.563983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075156.563988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075156.563991:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075156.563996:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2145 00000100:00080000:1.0:1769075156.564001:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075156.564010:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769075156.564012:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769075156.564022:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075156.564048:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617102976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075156.564108:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075156/real 1769075156] req@ffff9b0343dee680 x1855007617102464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075156.564148:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617102464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075156.564163:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075156.564169:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075156.564185:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617102464:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075156.564215:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102592:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075156.564246:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075156.564267:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075156/real 1769075156] req@ffff9b0343ded500 x1855007617102592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075156.564355:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617102592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075156.564390:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075156.564395:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075156.564399:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102592:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075156.564437:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102720:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075156.564469:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075156.564489:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075156/real 1769075156] req@ffff9b0343def480 x1855007617102720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075156.564505:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617102720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075156.564516:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075156.564520:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075156.564524:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102720:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075156.564531:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102848:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075156.564549:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075156.564560:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075156/real 1769075156] req@ffff9b0343defb80 x1855007617102848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075156.564583:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617102848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075156.564592:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075156.564595:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075156.564599:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102848:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075156.564605:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102976:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075156.564623:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075156.564632:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075156/real 1769075156] req@ffff9b0343dece00 x1855007617102976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075156.564646:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617102976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075156.564704:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075156.564709:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075156.564713:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617102976:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075157.075335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075157.460155:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075158.099313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075158.483590:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075159.123346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075159.507408:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075160.147348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075160.531652:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075161.171345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075161.555392:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075161.683342:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075161.683352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075161.683357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075161.683365:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2150 00000100:00080000:1.0:1769075161.683374:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075161.683380:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075161.683412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617103104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075161.683436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075161.683438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075161.683440:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075161.683443:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2150 00000100:00080000:1.0:1769075161.683445:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075161.683447:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075161.683454:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075161.683463:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617103232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075161.683470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075161.683474:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075161.683475:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075161.683478:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2150 00000100:00080000:1.0:1769075161.683480:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075161.683484:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617103104:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075161.683485:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075161.683491:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075161.683500:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617103360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075161.683507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075161.683510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075161.683511:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075161.683514:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2150 00000100:00080000:1.0:1769075161.683516:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075161.683518:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075161.683521:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075161.683528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617103488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075161.683534:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075161.683537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075161.683538:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075161.683540:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2150 00000100:00080000:1.0:1769075161.683543:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075161.683547:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769075161.683548:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769075161.683553:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075161.683561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedc00 x1855007617103616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075161.683608:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075161/real 1769075161] req@ffff9b0343deca80 x1855007617103104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075161.683630:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617103104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075161.683637:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075161.683641:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075161.683644:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617103104:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075161.683649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617103232:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075161.683666:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075161.683675:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075161/real 1769075161] req@ffff9b0343deed80 x1855007617103232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075161.683688:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617103232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075161.683696:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075161.683700:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075161.683703:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617103232:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075161.683728:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617103360:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075161.683747:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075161.683755:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075161/real 1769075161] req@ffff9b0343deea00 x1855007617103360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075161.683763:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617103360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075161.683773:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075161.683775:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075161.683777:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617103360:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075161.683781:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617103488:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075161.683791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075161.683797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075161/real 1769075161] req@ffff9b0343def800 x1855007617103488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075161.683804:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617103488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075161.683817:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075161.683820:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075161.683822:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617103488:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075161.683826:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617103616:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075161.683837:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075161.683844:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075161/real 1769075161] req@ffff9b0343dedc00 x1855007617103616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075161.683851:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedc00 x1855007617103616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075161.683856:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075161.683858:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075161.683860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617103616:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075162.195355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075162.580045:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075163.219372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075163.603586:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075164.243404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075164.627493:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075165.267384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075165.651483:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075166.291453:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075166.675456:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075166.803356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075166.803370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075166.803377:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075166.803390:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2155 00000100:00080000:1.0:1769075166.803406:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075166.803417:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075166.803475:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617103744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075166.803546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075166.803551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075166.803554:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075166.803560:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2155 00000100:00080000:1.0:1769075166.803567:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075166.803575:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075166.803589:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075166.803620:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617103872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075166.803621:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617103744:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075166.803663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075166.803667:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075166.803670:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075166.803676:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2155 00000100:00080000:1.0:1769075166.803682:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075166.803689:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075166.803696:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075166.803712:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617104000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075166.803726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769075166.803734:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075166.803736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075166.803744:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075166.803751:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2155 00000100:00080000:1.0:1769075166.803758:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075166.803763:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075166.803772:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075166.803790:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617104128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075166.803803:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075166.803812:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769075166.803813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075166/real 1769075166] req@ffff9b0343ded880 x1855007617103744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075166.803815:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075166.803831:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2155 00000100:00080000:1.0:1769075166.803845:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075166.803857:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075166.803864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617103744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769075166.803872:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769075166.803882:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075166.803903:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075166.803910:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617103744:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075166.803919:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617104256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075166.803995:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617103872:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075166.804052:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075166.804083:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075166/real 1769075166] req@ffff9b0343def100 x1855007617103872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075166.804117:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617103872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075166.804137:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075166.804146:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075166.804156:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617103872:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075166.804172:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104000:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075166.804216:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075166.804269:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075166/real 1769075166] req@ffff9b0343dee300 x1855007617104000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075166.804299:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617104000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075166.804312:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075166.804319:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075166.804328:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104000:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075166.804341:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104128:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075166.804377:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075166.804405:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075166/real 1769075166] req@ffff9b0343dec380 x1855007617104128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075166.804423:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617104128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075166.804437:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075166.804445:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075166.804453:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104128:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075166.804466:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104256:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075166.804517:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075166.804535:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075166/real 1769075166] req@ffff9b0343ded180 x1855007617104256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075166.804557:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617104256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075166.804574:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075166.804578:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075166.804583:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104256:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075167.315383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075167.699532:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075168.339379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075168.723500:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075169.363381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075169.747505:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075170.387402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075170.771487:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075171.411350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075171.795534:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075171.923345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075171.923358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075171.923363:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075171.923374:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2161 00000100:00080000:1.0:1769075171.923384:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075171.923391:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075171.923449:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617104384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075171.923479:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075171.923483:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075171.923490:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075171.923495:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2161 00000100:00080000:1.0:1769075171.923499:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075171.923509:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075171.923520:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075171.923534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617104512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075171.923547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769075171.923549:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617104384:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075171.923553:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075171.923558:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075171.923565:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2161 00000100:00080000:1.0:1769075171.923569:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075171.923572:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075171.923577:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075171.923591:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617104640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075171.923601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075171.923605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075171.923607:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075171.923611:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2161 00000100:00080000:1.0:1769075171.923614:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075171.923618:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075171.923623:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075171.923634:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075171.923638:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617104768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075171.923660:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075171.923664:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075171.923666:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075171.923670:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2161 00000100:00080000:1.0:1769075171.923674:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075171.923678:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075171.923683:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075171.923696:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075171/real 1769075171] req@ffff9b0343dec000 x1855007617104384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075171.923697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617104896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075171.923745:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617104384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075171.923761:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075171.923768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075171.923823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617104384:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075171.923878:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104512:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075171.923913:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075171.923930:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075171/real 1769075171] req@ffff9b0343dec700 x1855007617104512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075171.923946:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617104512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075171.923956:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075171.923960:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075171.923965:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104512:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075171.923972:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104640:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075171.924008:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075171.924017:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075171/real 1769075171] req@ffff9b0343dedf80 x1855007617104640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075171.924036:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617104640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075171.924043:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075171.924046:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075171.924048:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104640:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075171.924054:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104768:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075171.924068:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075171.924076:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075171/real 1769075171] req@ffff9b0343ded180 x1855007617104768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075171.924086:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617104768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075171.924093:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075171.924096:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075171.924099:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104768:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075171.924105:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104896:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075171.924129:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075171.924153:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075171/real 1769075171] req@ffff9b0343dec380 x1855007617104896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075171.924173:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617104896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075171.924186:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075171.924191:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075171.924196:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617104896:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075172.435367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075172.820065:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075173.459485:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075173.843512:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075174.483427:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075174.867827:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075175.507456:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075175.891940:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075176.531349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075176.915857:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075177.043408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075177.043421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075177.043429:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075177.043445:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2166 00000100:00080000:1.0:1769075177.043462:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075177.043473:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075177.043532:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617105024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075177.043576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075177.043581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075177.043591:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075177.043597:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2166 00000100:00080000:1.0:1769075177.043603:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075177.043608:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075177.043623:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075177.043643:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617105152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075177.043655:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617105024:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075177.043664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075177.043671:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075177.043674:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075177.043680:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2166 00000100:00080000:1.0:1769075177.043686:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075177.043691:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075177.043697:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075177.043714:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617105280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075177.043726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075177.043731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075177.043733:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075177.043737:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2166 00000100:00080000:1.0:1769075177.043743:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769075177.043750:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075177.043755:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075177.043765:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075177.043781:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedc00 x1855007617105408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075177.043794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075177.043798:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075177.043802:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075177.043807:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2166 00000100:00080000:1.0:1769075177.043814:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075177.043818:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9b0343dee300 x1855007617105024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075177.043822:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075177.043833:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075177.043847:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617105536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075177.043905:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617105024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075177.043929:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075177.043938:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075177.043947:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617105024:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075177.043980:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105152:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075177.044015:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075177.044033:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9b0343def100 x1855007617105152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075177.044050:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617105152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075177.044062:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075177.044067:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075177.044071:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105152:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075177.044079:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105280:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075177.044099:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075177.044110:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9b0343ded880 x1855007617105280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075177.044125:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617105280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075177.044134:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075177.044138:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075177.044142:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105280:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075177.044149:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105408:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075177.044167:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075177.044178:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9b0343dedc00 x1855007617105408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075177.044192:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedc00 x1855007617105408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075177.044201:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075177.044205:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075177.044210:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105408:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075177.044217:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105536:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075177.044282:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075177.044304:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075177/real 1769075177] req@ffff9b0343def800 x1855007617105536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075177.044359:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617105536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075177.044376:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075177.044382:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075177.044389:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105536:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075177.555419:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075177.939824:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075178.579434:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075178.963538:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075179.603454:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075179.987533:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075180.627339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075181.011536:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075181.651390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075182.035499:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075182.163411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075182.163425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075182.163432:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075182.163444:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2171 00000100:00080000:2.0:1769075182.163458:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075182.163467:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075182.163521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617105664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075182.163578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075182.163586:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075182.163590:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075182.163598:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2171 00000100:00080000:2.0:1769075182.163607:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075182.163617:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075182.163633:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075182.163655:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617105792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075182.163666:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617105664:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075182.163679:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075182.163686:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075182.163689:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075182.163696:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2171 00000100:00080000:2.0:1769075182.163704:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075182.163709:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075182.163715:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075182.163732:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617105920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075182.163744:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075182.163748:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075182.163752:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075182.163756:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2171 00000100:00080000:2.0:1769075182.163761:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075182.163765:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075182.163771:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075182.163784:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7800 x1855007617106048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075182.163804:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075182.163810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075182.163815:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075182.163824:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2171 00000100:00080000:2.0:1769075182.163832:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075182.163838:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075182.163843:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075182.163861:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105792:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075182.163872:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617106176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075182.163934:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105920:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075182.163990:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106048:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075182.164055:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106176:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075182.675305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075183.059466:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075183.699384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075184.083430:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075184.723323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075185.107347:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769075185.235629:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:3.0:1769075185.235669:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769075185.235686:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075185.235724:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075185.235734:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075185.235743:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075185.235762:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769075185.235801:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075182/real 1769075185] req@ffff9b03482c4a80 x1855007617106176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075185.235830:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617106176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075185.235840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075185.235844:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075185.235848:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106176:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075185.235857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075182/real 1769075185] req@ffff9b03482c6300 x1855007617105792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075185.235870:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617105792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075185.235879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075185.235884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075185.235888:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105792:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075185.235895:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075182/real 1769075185] req@ffff9b03482c6d80 x1855007617105920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075185.235909:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617105920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075185.235916:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075185.235920:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075185.235923:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617105920:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075185.235945:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075182/real 1769075185] req@ffff9b03482c7800 x1855007617106048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075185.235959:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7800 x1855007617106048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075185.235968:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075185.235972:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075185.235977:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106048:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075185.235984:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075182/real 1769075185] req@ffff9b03482c5180 x1855007617105664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075185.236016:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617105664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075185.236024:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075185.236035:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075185.236038:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617105664:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075185.747366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075186.131436:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075186.771303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075187.155401:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075187.283414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075187.283446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075187.283452:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075187.283475:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2176 00000100:00080000:2.0:1769075187.283489:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075187.283504:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075187.283550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617106304/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075187.283575:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075187.283582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075187.283583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075187.283587:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2176 00000100:00080000:2.0:1769075187.283591:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075187.283594:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075187.283603:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075187.283614:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7800 x1855007617106432/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075187.283623:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617106304:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075187.283626:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075187.283632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075187.283641:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075187.283646:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2176 00000100:00080000:2.0:1769075187.283648:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075187.283652:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075187.283656:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075187.283665:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617106560/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075187.283673:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075187.283676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075187.283678:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075187.283680:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2176 00000100:00080000:2.0:1769075187.283691:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075187.283693:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075187.283697:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075187.283706:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075187.283707:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617106688/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075187.283717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075187.283720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075187.283722:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075187.283725:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2176 00000100:00080000:2.0:1769075187.283729:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075187.283731:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075187.283735:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075187.283743:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617106816/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075187.283752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075187/real 1769075187] req@ffff9b03482c5180 x1855007617106304/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075187.283771:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617106304/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075187.283790:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075187.283794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075187.283797:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617106304:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075187.283818:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106432:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075187.283836:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075187.283855:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075187/real 1769075187] req@ffff9b03482c7800 x1855007617106432/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075187.283864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7800 x1855007617106432/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075187.283871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075187.283873:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075187.283876:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106432:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075187.283880:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106560:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075187.283891:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075187.283897:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075187/real 1769075187] req@ffff9b03482c6d80 x1855007617106560/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075187.283906:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617106560/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075187.283912:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075187.283914:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075187.283916:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106560:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075187.283921:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106688:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075187.283948:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075187.283967:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075187/real 1769075187] req@ffff9b03482c6300 x1855007617106688/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075187.283983:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617106688/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075187.283991:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075187.283995:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075187.283999:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106688:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075187.284006:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106816:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075187.284027:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075187.284040:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075187/real 1769075187] req@ffff9b03482c4a80 x1855007617106816/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075187.284053:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617106816/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075187.284062:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075187.284066:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075187.284069:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617106816:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075187.795298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075188.179420:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075188.819350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075189.203361:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075189.843332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075190.227475:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075190.867350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075191.251514:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075191.891317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075192.275452:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075192.403386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075192.403411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075192.403417:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075192.403430:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2181 00000100:00080000:2.0:1769075192.403444:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075192.403454:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075192.403515:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617106944/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075192.403560:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075192.403568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075192.403572:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075192.403580:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2181 00000100:00080000:2.0:1769075192.403588:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075192.403595:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075192.403623:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075192.403651:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617107072/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075192.403665:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617106944:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075192.403695:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075192.403706:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075192.403710:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075192.403718:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2181 00000100:00080000:2.0:1769075192.403725:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075192.403731:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075192.403740:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075192.403772:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617107200/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075192.403786:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075192.403790:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075192.403796:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769075192.403800:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075192.403806:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2181 00000100:00080000:2.0:1769075192.403814:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075192.403820:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075192.403827:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075192.403875:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075192/real 1769075192] req@ffff9b03482c4380 x1855007617106944/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075192.403879:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617107328/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075192.403926:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075192.403931:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075192.403936:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075192.403938:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617106944/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075192.403946:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2181 00000100:00080000:2.0:1769075192.403956:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075192.403968:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:3.0:1769075192.403970:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769075192.403975:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075192.403989:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617107456/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075192.404009:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075192.404019:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617106944:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075192.404089:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107072:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075192.404126:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075192.404143:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075192/real 1769075192] req@ffff9b03482c5500 x1855007617107072/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075192.404160:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617107072/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075192.404171:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075192.404175:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075192.404180:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107072:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075192.404187:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107200:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075192.404207:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075192.404218:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075192/real 1769075192] req@ffff9b03482c4a80 x1855007617107200/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075192.404292:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617107200/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075192.404303:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075192.404307:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075192.404311:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107200:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075192.404319:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107328:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075192.404343:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075192.404356:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075192/real 1769075192] req@ffff9b03482c6300 x1855007617107328/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075192.404370:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617107328/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075192.404380:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075192.404383:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075192.404387:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107328:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075192.404394:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107456:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075192.404412:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075192.404422:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075192/real 1769075192] req@ffff9b03482c6d80 x1855007617107456/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075192.404436:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617107456/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075192.404453:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075192.404457:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075192.404461:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107456:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075192.915346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075193.299630:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075193.939396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075194.323762:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075194.963336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075195.347737:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075195.987424:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075196.371595:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075197.011422:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075197.395581:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075197.523482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075197.523499:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075197.523506:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075197.523524:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2186 00000100:00080000:1.0:1769075197.523541:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075197.523552:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075197.523612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617107584/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075197.523655:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075197.523661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075197.523664:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075197.523669:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2186 00000100:00080000:1.0:1769075197.523674:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075197.523678:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075197.523692:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075197.523709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617107712/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075197.523727:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075197.523738:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1769075197.523742:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617107584:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075197.523743:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075197.523761:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2186 00000100:00080000:1.0:1769075197.523768:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075197.523773:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075197.523780:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075197.523808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617107840/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075197.523823:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075197.523828:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075197.523832:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075197.523838:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2186 00000100:00080000:1.0:1769075197.523854:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075197.523858:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075197.523866:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075197.523878:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075197.523892:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617107968/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075197.523912:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075197.523917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075197.523919:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075197.523925:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2186 00000100:00080000:1.0:1769075197.523935:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075197.523939:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075197.523947:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075197.523948:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075197/real 1769075197] req@ffff9b0343deea00 x1855007617107584/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075197.524032:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617108096/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075197.524041:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617107584/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075197.524129:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075197.524140:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075197.524147:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617107584:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075197.524186:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107712:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075197.524229:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075197.524272:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075197/real 1769075197] req@ffff9b0343deed80 x1855007617107712/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075197.524375:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617107712/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075197.524389:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075197.524394:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075197.524400:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107712:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075197.524411:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107840:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075197.524445:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075197.524460:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075197/real 1769075197] req@ffff9b0343deca80 x1855007617107840/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075197.524472:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617107840/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075197.524480:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075197.524484:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075197.524487:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107840:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075197.524493:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107968:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075197.524511:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075197.524519:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075197/real 1769075197] req@ffff9b0343dece00 x1855007617107968/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075197.524530:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617107968/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075197.524538:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075197.524541:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075197.524545:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617107968:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075197.524550:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108096:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075197.524564:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075197.524572:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075197/real 1769075197] req@ffff9b0343defb80 x1855007617108096/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075197.524582:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617108096/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075197.524590:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075197.524593:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075197.524596:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108096:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075198.035369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075198.419584:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075199.059365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075199.443554:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075200.083414:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075200.467517:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075201.107353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075201.491379:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075202.131355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075202.515435:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075202.643463:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075202.643476:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075202.643482:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075202.643496:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2191 00000100:00080000:1.0:1769075202.643512:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075202.643536:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075202.643590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617108224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075202.643629:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075202.643635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075202.643637:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075202.643643:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2191 00000100:00080000:1.0:1769075202.643648:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075202.643653:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075202.643665:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075202.643697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617108352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075202.643744:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769075202.643746:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617108224:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075202.643758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075202.643763:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075202.643771:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2191 00000100:00080000:1.0:1769075202.643775:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075202.643780:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075202.643787:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075202.643810:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617108480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075202.643822:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075202.643827:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075202.643830:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075202.643834:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2191 00000100:00080000:1.0:1769075202.643840:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075202.643847:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075202.643868:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075202.643877:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075202.643920:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617108608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075202.643940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075202.643947:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075202.643951:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769075202.643956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075202/real 1769075202] req@ffff9b0343def480 x1855007617108224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075202.643981:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2191 00000100:00080000:1.0:1769075202.643997:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075202.644008:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075202.644019:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075202.644020:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617108224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075202.644060:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769075202.644066:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617108736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075202.644072:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075202.644085:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617108224:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075202.644122:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108352:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075202.644154:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075202.644168:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075202/real 1769075202] req@ffff9b0343ded500 x1855007617108352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075202.644182:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617108352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075202.644191:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075202.644195:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075202.644201:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108352:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075202.644208:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108480:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075202.644228:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075202.644239:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075202/real 1769075202] req@ffff9b0343dee680 x1855007617108480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075202.644252:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617108480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075202.644322:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075202.644333:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075202.644341:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108480:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075202.644356:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108608:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075202.644402:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075202.644429:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075202/real 1769075202] req@ffff9b0343defb80 x1855007617108608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075202.644456:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617108608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075202.644471:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075202.644478:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075202.644483:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108608:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075202.644492:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108736:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075202.644544:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075202.644560:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075202/real 1769075202] req@ffff9b0343dece00 x1855007617108736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075202.644574:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617108736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075202.644584:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075202.644587:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075202.644592:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108736:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075203.155428:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075203.539482:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075204.179377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075204.563558:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075205.203458:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075205.587370:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075206.227322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075206.611398:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075207.251349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075207.635502:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075207.763413:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075207.763429:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075207.763436:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075207.763450:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2196 00000100:00080000:3.0:1769075207.763481:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075207.763494:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075207.763538:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617108864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075207.763575:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075207.763579:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075207.763581:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075207.763585:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2196 00000100:00080000:3.0:1769075207.763589:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075207.763593:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075207.763604:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075207.763617:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617108992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075207.763629:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075207.763632:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075207.763634:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075207.763639:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2196 00000100:00080000:3.0:1769075207.763642:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075207.763649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617108864:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075207.763651:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075207.763663:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075207.763677:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617109120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075207.763688:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075207.763692:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075207.763694:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075207.763699:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2196 00000100:00080000:3.0:1769075207.763714:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075207.763719:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075207.763725:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075207.763732:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075207.763739:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617109248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075207.763751:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075207.763754:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075207.763756:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075207.763760:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2196 00000100:00080000:3.0:1769075207.763763:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075207.763767:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075207.763772:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075207.763783:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075207/real 1769075207] req@ffff9b0352a18a80 x1855007617108864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075207.763788:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617109376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075207.763864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617108864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075207.763877:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075207.763883:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075207.763887:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617108864:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075207.763896:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108992:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075207.763922:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075207.763935:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075207/real 1769075207] req@ffff9b0352a1b100 x1855007617108992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075207.763948:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617108992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075207.763956:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075207.763960:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075207.763963:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617108992:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075207.763989:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109120:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075207.764008:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075207.764017:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075207/real 1769075207] req@ffff9b0352a1ad80 x1855007617109120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075207.764047:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617109120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075207.764055:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075207.764058:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075207.764061:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109120:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075207.764067:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109248:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075207.764085:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075207.764095:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075207/real 1769075207] req@ffff9b0352a1b480 x1855007617109248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075207.764106:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617109248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075207.764114:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075207.764117:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075207.764120:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109248:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075207.764125:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109376:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075207.764139:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075207.764147:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075207/real 1769075207] req@ffff9b0352a1a680 x1855007617109376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075207.764165:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617109376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075207.764172:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075207.764175:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075207.764178:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109376:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075208.275350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075208.659506:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075209.299386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075209.683531:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075210.323364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075210.707486:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075211.347383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075211.731412:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075212.371385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075212.755483:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075212.883397:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075212.883410:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075212.883417:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075212.883429:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2202 00000100:00080000:3.0:1769075212.883442:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075212.883451:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075212.883496:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617109504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075212.883535:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075212.883551:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075212.883553:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075212.883558:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2202 00000100:00080000:3.0:1769075212.883563:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075212.883567:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075212.883578:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075212.883592:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617109632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075212.883616:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075212.883623:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075212.883627:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075212.883637:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2202 00000100:00080000:3.0:1769075212.883644:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075212.883652:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075212.883662:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075212.883667:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617109504:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075212.883690:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617109760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075212.883773:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075212.883782:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075212.883786:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769075212.883795:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075212.883820:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2202 00000100:00080000:3.0:1769075212.883832:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075212.883840:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075212.883854:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075212.883881:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617109888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075212.883888:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075212/real 1769075212] req@ffff9b0352a19f80 x1855007617109504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075212.883937:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075212.883947:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769075212.883948:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617109504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075212.883954:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075212.883969:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075212.883978:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2202 00000100:00080000:2.0:1769075212.883982:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769075212.883991:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075212.883993:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617109504:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075212.884006:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075212.884013:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109632:192.168.204.160@tcp:38:kworker.0 00000008:00100000:3.0:1769075212.884020:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075212.884048:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617110016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075212.884060:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075212.884091:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075212/real 1769075212] req@ffff9b0352a1a300 x1855007617109632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075212.884113:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617109632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075212.884140:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075212.884144:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075212.884148:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109632:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075212.884176:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109760:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075212.884206:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075212.884217:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075212/real 1769075212] req@ffff9b0352a18e00 x1855007617109760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075212.884282:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617109760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075212.884294:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075212.884298:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075212.884302:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109760:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075212.884310:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109888:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075212.884335:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075212.884349:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075212/real 1769075212] req@ffff9b0352a18700 x1855007617109888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075212.884364:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617109888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075212.884373:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075212.884377:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075212.884382:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617109888:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075212.884389:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110016:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075212.884407:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075212.884417:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075212/real 1769075212] req@ffff9b0352a1aa00 x1855007617110016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075212.884431:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617110016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075212.884440:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075212.884443:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075212.884447:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110016:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075213.395417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075213.779404:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075214.419390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075214.803456:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075215.443379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075215.827407:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075216.467469:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075216.851535:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075217.491362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075217.875434:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075218.003478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.003493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075218.003500:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.003515:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2207 00000100:00080000:1.0:1769075218.003530:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.003540:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075218.003586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617110144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075218.003654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.003660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075218.003664:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.003671:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2207 00000100:00080000:1.0:1769075218.003679:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.003696:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075218.003710:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075218.003729:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617110272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075218.003750:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617110144:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075218.003752:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.003772:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075218.003776:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.003783:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2207 00000100:00080000:1.0:1769075218.003788:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.003792:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075218.003800:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075218.003830:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617110400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075218.003842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.003847:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075218.003850:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.003856:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2207 00000100:00080000:1.0:1769075218.003861:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075218.003865:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075218.003872:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075218.003894:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617110528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075218.003914:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075218.003935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075218.003944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075218.003949:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075218.003961:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2207 00000100:00080000:1.0:1769075218.003973:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075218.003994:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075218/real 1769075218] req@ffff9b0343deca80 x1855007617110144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075218.004001:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075218.004021:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075218.004049:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedc00 x1855007617110656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075218.004062:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617110144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075218.004143:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075218.004151:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075218.004160:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617110144:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075218.004202:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110272:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075218.004284:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075218.004303:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075218/real 1769075218] req@ffff9b0343deed80 x1855007617110272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075218.004320:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617110272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075218.004331:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075218.004335:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075218.004339:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110272:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075218.004347:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110400:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075218.004370:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075218.004380:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075218/real 1769075218] req@ffff9b0343deea00 x1855007617110400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075218.004394:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617110400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075218.004403:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075218.004407:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075218.004410:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110400:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075218.004417:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110528:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075218.004435:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075218.004445:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075218/real 1769075218] req@ffff9b0343def800 x1855007617110528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075218.004487:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617110528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075218.004496:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075218.004500:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075218.004503:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110528:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075218.004511:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110656:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075218.004531:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075218.004542:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075218/real 1769075218] req@ffff9b0343dedc00 x1855007617110656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075218.004555:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedc00 x1855007617110656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075218.004564:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075218.004568:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075218.004572:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110656:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075218.515392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075218.899358:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075219.539326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075219.923308:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075220.563391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075220.947424:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075221.587418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075221.971428:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075222.611407:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075222.995407:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075223.123318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075223.123327:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075223.123331:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075223.123339:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2212 00000100:00080000:1.0:1769075223.123347:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075223.123353:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075223.123385:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617110784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075223.123408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075223.123411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075223.123412:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075223.123415:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2212 00000100:00080000:1.0:1769075223.123418:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075223.123421:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075223.123429:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075223.123439:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617110912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075223.123440:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617110784:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075223.123457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075223.123462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075223.123464:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075223.123469:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2212 00000100:00080000:1.0:1769075223.123474:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075223.123479:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075223.123485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075223.123496:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617111040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075223.123506:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769075223.123506:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075223.123512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075223.123515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075223.123521:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2212 00000100:00080000:1.0:1769075223.123525:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075223.123530:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075223.123535:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075223.123556:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075223/real 1769075223] req@ffff9b0343ded880 x1855007617110784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075223.123562:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617111168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075223.123576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769075223.123578:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617110784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075223.123585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075223.123589:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075223.123600:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2212 00000100:00080000:2.0:1769075223.123603:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075223.123605:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075223.123610:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769075223.123614:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075223.123617:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617110784:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769075223.123623:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075223.123638:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617111296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075223.123642:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110912:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075223.123669:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075223.123680:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075223/real 1769075223] req@ffff9b0343def100 x1855007617110912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075223.123689:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617110912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075223.123703:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075223.123706:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075223.123708:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617110912:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075223.123712:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111040:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075223.123724:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075223.123730:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075223/real 1769075223] req@ffff9b0343dee300 x1855007617111040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075223.123738:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617111040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075223.123743:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075223.123745:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075223.123747:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111040:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075223.123751:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111168:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075223.123761:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075223.123767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075223/real 1769075223] req@ffff9b0343dec380 x1855007617111168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075223.123775:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617111168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075223.123783:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075223.123786:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075223.123789:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111168:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075223.123805:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111296:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075223.123826:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075223.123837:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075223/real 1769075223] req@ffff9b0343ded180 x1855007617111296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075223.123849:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617111296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075223.123854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075223.123856:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075223.123858:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111296:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075223.635314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075224.019335:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075224.659356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075225.043408:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075225.683319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075226.067425:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075226.707324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075227.091353:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075227.731297:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075228.115436:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075228.243436:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075228.243449:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075228.243481:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075228.243496:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2217 00000100:00080000:3.0:1769075228.243509:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075228.243518:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075228.243567:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617111424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075228.243604:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075228.243609:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075228.243612:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075228.243617:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2217 00000100:00080000:3.0:1769075228.243622:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075228.243628:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075228.243651:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075228.243669:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617111552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075228.243677:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617111424:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075228.243717:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075228.243741:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075228.243744:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075228.243750:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2217 00000100:00080000:3.0:1769075228.243754:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075228.243759:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075228.243766:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075228.243780:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617111680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075228.243792:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075228.243796:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075228.243798:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075228.243813:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2217 00000100:00080000:3.0:1769075228.243818:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769075228.243827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075228.243828:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075228.243849:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075228.243869:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617111808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075228.243882:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075228.243886:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075228.243889:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075228.243899:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2217 00000100:00100000:2.0:1769075228.243903:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075228/real 1769075228] req@ffff9b0352a1aa00 x1855007617111424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075228.243904:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075228.243912:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075228.243920:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075228.243938:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617111936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075228.243959:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617111424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075228.243984:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075228.243992:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075228.244000:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617111424:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075228.244049:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111552:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075228.244086:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075228.244103:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075228/real 1769075228] req@ffff9b0352a18700 x1855007617111552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075228.244120:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617111552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075228.244131:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075228.244136:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075228.244140:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111552:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075228.244148:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111680:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075228.244168:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075228.244180:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075228/real 1769075228] req@ffff9b0352a18e00 x1855007617111680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075228.244194:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617111680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075228.244213:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075228.244217:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075228.244221:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111680:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075228.244229:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111808:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075228.244254:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075228.244333:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075228/real 1769075228] req@ffff9b0352a1a300 x1855007617111808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075228.244358:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617111808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075228.244373:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075228.244379:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075228.244384:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111808:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075228.244392:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111936:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075228.244441:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075228.244496:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075228/real 1769075228] req@ffff9b0352a19f80 x1855007617111936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075228.244518:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617111936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075228.244527:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075228.244531:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075228.244535:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617111936:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075228.755425:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075229.139760:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075229.779382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075230.163681:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075230.803386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075231.187647:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075231.827408:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075232.211860:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075232.851334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075233.235346:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075233.363357:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075233.363383:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075233.363390:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075233.363416:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2222 00000100:00080000:3.0:1769075233.363425:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075233.363465:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075233.363496:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617112064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075233.363524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075233.363526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075233.363528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075233.363531:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2222 00000100:00080000:3.0:1769075233.363534:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075233.363538:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075233.363545:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075233.363555:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617112064:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075233.363556:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617112192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075233.363570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075233.363572:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075233.363573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075233.363576:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2222 00000100:00080000:3.0:1769075233.363579:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075233.363581:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075233.363584:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075233.363592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617112320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075233.363598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075233.363600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075233.363601:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075233.363604:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2222 00000100:00080000:3.0:1769075233.363608:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075233.363610:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075233.363614:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075233.363621:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617112448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075233.363628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769075233.363629:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075233.363639:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075233.363642:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075233.363646:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2222 00000100:00080000:3.0:1769075233.363649:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075233.363651:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075233.363655:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075233.363667:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617112576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075233.363671:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075233/real 1769075233] req@ffff9b0352a1a680 x1855007617112064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075233.363696:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617112064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075233.363708:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075233.363713:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075233.363718:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617112064:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075233.363741:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112192:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075233.363778:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075233.363788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075233/real 1769075233] req@ffff9b0352a1b480 x1855007617112192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075233.363797:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617112192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075233.363802:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075233.363804:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075233.363807:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112192:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075233.363810:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112320:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075233.363823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075233.363828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075233/real 1769075233] req@ffff9b0352a1ad80 x1855007617112320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075233.363836:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617112320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075233.363840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075233.363842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075233.363844:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112320:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075233.363848:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112448:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075233.363857:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075233.363862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075233/real 1769075233] req@ffff9b0352a1b100 x1855007617112448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075233.363873:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617112448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075233.363881:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075233.363884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075233.363888:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112448:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075233.363894:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112576:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075233.363913:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075233.363923:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075233/real 1769075233] req@ffff9b0352a18a80 x1855007617112576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075233.363940:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617112576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075233.363946:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075233.363948:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075233.363950:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112576:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075233.875332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075234.259384:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075234.899365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075235.283451:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075235.923401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075236.307548:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075236.947434:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075237.331439:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075237.971443:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075238.355379:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075238.483353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075238.483362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075238.483366:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075238.483375:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2227 00000100:00080000:0.0:1769075238.483383:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075238.483388:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075238.483420:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617112704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075238.483460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075238.483465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075238.483467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075238.483472:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2227 00000100:00080000:0.0:1769075238.483476:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075238.483481:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075238.483491:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075238.483508:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617112704:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075238.483511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617112832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075238.483524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075238.483527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075238.483529:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075238.483532:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2227 00000100:00080000:0.0:1769075238.483535:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075238.483538:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075238.483543:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075238.483553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617112960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075238.483560:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075238.483563:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075238.483565:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075238.483568:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2227 00000100:00080000:0.0:1769075238.483571:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075238.483576:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769075238.483580:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769075238.483582:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075238.483593:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617113088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075238.483601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075238.483603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075238.483605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075238.483609:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2227 00000100:00080000:0.0:1769075238.483612:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075238.483615:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075238.483619:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075238.483624:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075238/real 1769075238] req@ffff9b0365a39180 x1855007617112704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075238.483630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617113216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075238.483649:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617112704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075238.483660:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075238.483665:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075238.483668:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617112704:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075238.483689:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112832:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075238.483708:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075238.483716:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075238/real 1769075238] req@ffff9b0365a3ad80 x1855007617112832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075238.483724:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617112832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075238.483730:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075238.483732:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075238.483735:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112832:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075238.483738:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112960:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075238.483769:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075238.483777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075238/real 1769075238] req@ffff9b0365a3b100 x1855007617112960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075238.483785:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617112960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075238.483795:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075238.483798:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075238.483800:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617112960:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075238.483804:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113088:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075238.483816:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075238.483828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075238/real 1769075238] req@ffff9b0365a3bb80 x1855007617113088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075238.483841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617113088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075238.483850:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075238.483854:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075238.483858:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113088:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075238.483866:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113216:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075238.483890:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075238.483901:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075238/real 1769075238] req@ffff9b0365a39500 x1855007617113216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075238.483910:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617113216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075238.483918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075238.483921:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075238.483926:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113216:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075238.995387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075239.379405:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075240.019327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075240.403464:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075241.043323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075241.427376:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075242.067328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075242.451416:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075243.091325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075243.475422:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075243.603314:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075243.603324:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075243.603327:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075243.603336:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2232 00000100:00080000:0.0:1769075243.603345:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075243.603351:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075243.603380:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617113344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075243.603405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075243.603408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075243.603410:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075243.603413:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2232 00000100:00080000:0.0:1769075243.603415:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075243.603418:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075243.603424:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075243.603433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b800 x1855007617113472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075243.603441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075243.603443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075243.603446:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075243.603448:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617113344:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075243.603450:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2232 00000100:00080000:0.0:1769075243.603455:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075243.603458:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075243.603462:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075243.603474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617113600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075243.603484:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075243.603488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075243.603490:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075243.603493:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2232 00000100:00080000:0.0:1769075243.603499:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075243.603503:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769075243.603507:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769075243.603512:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075243.603528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617113728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075243.603540:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075243.603543:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075243/real 1769075243] req@ffff9b0365a39c00 x1855007617113344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075243.603544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075243.603547:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075243.603554:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2232 00000100:00080000:0.0:1769075243.603560:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075243.603565:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075243.603569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075243.603578:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617113344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075243.603580:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39f80 x1855007617113856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075243.603614:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075243.603619:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075243.603623:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617113344:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075243.603642:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113472:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075243.603661:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075243.603670:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075243/real 1769075243] req@ffff9b0365a3b800 x1855007617113472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075243.603679:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b800 x1855007617113472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075243.603684:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075243.603687:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075243.603689:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113472:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075243.603693:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113600:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075243.603703:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075243.603709:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075243/real 1769075243] req@ffff9b0365a3aa00 x1855007617113600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075243.603716:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617113600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075243.603721:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075243.603723:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075243.603725:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113600:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075243.603728:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113728:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075243.603738:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075243.603744:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075243/real 1769075243] req@ffff9b0365a3b480 x1855007617113728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075243.603751:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617113728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075243.603756:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075243.603758:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075243.603759:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113728:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075243.603763:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113856:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075243.603771:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075243.603776:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075243/real 1769075243] req@ffff9b0365a39f80 x1855007617113856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075243.603783:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39f80 x1855007617113856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075243.603789:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075243.603791:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075243.603793:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617113856:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075244.115298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075244.499359:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075245.139300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075245.523347:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075246.163317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075246.547417:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075247.187292:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075247.571388:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075248.211309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075248.595401:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075248.723333:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075248.723345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075248.723349:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075248.723359:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2237 00000100:00080000:0.0:1769075248.723369:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075248.723376:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075248.723412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617113984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075248.723443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075248.723447:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075248.723449:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075248.723454:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2237 00000100:00080000:0.0:1769075248.723457:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075248.723461:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075248.723470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075248.723484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617114112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075248.723497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075248.723501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075248.723504:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075248.723507:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617113984:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075248.723510:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2237 00000100:00080000:0.0:1769075248.723517:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075248.723522:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075248.723527:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075248.723539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38000 x1855007617114240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075248.723549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075248.723552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075248.723554:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075248.723558:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2237 00000100:00080000:0.0:1769075248.723572:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075248.723575:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075248.723581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075248.723592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39f80 x1855007617114368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075248.723602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075248.723605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075248.723607:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075248.723612:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2237 00000100:00080000:0.0:1769075248.723615:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075248.723619:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075248.723625:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075248.723627:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114112:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075248.723638:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617114496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075248.723663:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114240:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075248.723685:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114368:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075248.723714:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114496:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075249.235356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075249.619407:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075250.259386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075250.643381:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075251.283354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075251.667431:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075251.795567:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769075251.795605:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075251.795623:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075251.795670:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075251.795680:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075251.795691:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075251.795702:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769075251.795716:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075248/real 1769075251] req@ffff9b0365a38a80 x1855007617114112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075251.795745:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617114112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075251.795757:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075251.795764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075251.795768:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114112:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075251.795777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075248/real 1769075251] req@ffff9b0365a38000 x1855007617114240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075251.795790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38000 x1855007617114240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075251.795797:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075251.795801:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075251.795804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114240:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075251.795811:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075248/real 1769075251] req@ffff9b0365a39f80 x1855007617114368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075251.795821:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39f80 x1855007617114368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075251.795828:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075251.795848:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075251.795851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114368:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075251.795857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075248/real 1769075251] req@ffff9b0365a38e00 x1855007617113984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075251.795871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617113984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075251.795879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075251.795884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075251.795887:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617113984:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075251.795915:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075248/real 1769075251] req@ffff9b0365a3b480 x1855007617114496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075251.795928:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617114496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075251.795936:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075251.795940:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075251.795943:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114496:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075252.307327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075252.691415:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075253.331328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075253.715398:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075253.843386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075253.843397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075253.843402:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075253.843412:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2243 00000100:00080000:3.0:1769075253.843423:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075253.843431:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075253.843465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617114624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075253.843497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075253.843501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075253.843503:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075253.843507:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2243 00000100:00080000:3.0:1769075253.843511:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075253.843515:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075253.843525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075253.843540:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617114752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075253.843551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075253.843556:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769075253.843558:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617114624:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075253.843559:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075253.843569:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2243 00000100:00080000:3.0:1769075253.843573:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075253.843577:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075253.843583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075253.843597:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617114880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075253.843606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075253.843610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075253.843613:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075253.843617:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2243 00000100:00080000:3.0:1769075253.843621:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075253.843625:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769075253.843631:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769075253.843635:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075253.843650:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617115008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075253.843661:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075253.843665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075253.843667:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075253.843672:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2243 00000100:00080000:3.0:1769075253.843677:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075253.843678:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075253/real 1769075253] req@ffff9b0352a19c00 x1855007617114624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075253.843686:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075253.843696:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075253.843710:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617114624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075253.843712:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617115136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075253.843747:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075253.843754:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075253.843758:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617114624:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075253.843786:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114752:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075253.843817:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075253.843831:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075253/real 1769075253] req@ffff9b0352a18380 x1855007617114752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075253.843843:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617114752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075253.843851:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075253.843854:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075253.843858:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114752:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075253.843864:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114880:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075253.843883:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075253.843894:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075253/real 1769075253] req@ffff9b0352a1b800 x1855007617114880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075253.843905:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617114880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075253.843913:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075253.843916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075253.843920:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617114880:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075253.843926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115008:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075253.843947:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075253.843966:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075253/real 1769075253] req@ffff9b0352a18000 x1855007617115008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075253.843993:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617115008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075253.844001:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075253.844004:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075253.844007:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115008:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075253.844014:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115136:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075253.844052:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075253.844064:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075253/real 1769075253] req@ffff9b0352a19500 x1855007617115136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075253.844077:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617115136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075253.844090:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075253.844093:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075253.844096:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115136:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075254.355290:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075254.739340:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075255.379295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075255.763441:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075256.403292:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075256.787327:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075257.427295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075257.811392:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075258.451306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075258.835351:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075258.963407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075258.963425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075258.963432:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075258.963449:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2248 00000100:00080000:3.0:1769075258.963466:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075258.963478:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075258.963537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617115264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075258.963576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075258.963580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075258.963582:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075258.963586:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2248 00000100:00080000:3.0:1769075258.963591:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075258.963595:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075258.963606:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075258.963618:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617115392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075258.963629:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075258.963634:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075258.963636:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075258.963640:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2248 00000100:00080000:3.0:1769075258.963643:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075258.963647:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075258.963653:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075258.963662:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617115520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075258.963677:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617115264:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075258.963678:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075258.963686:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075258.963689:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075258.963697:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2248 00000100:00080000:3.0:1769075258.963701:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075258.963705:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075258.963711:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075258.963721:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617115648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075258.963729:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075258.963750:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075258.963752:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075258.963758:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2248 00000100:00080000:3.0:1769075258.963771:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075258.963775:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075258.963781:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075258.963790:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075258.963804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617115776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075258.963841:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075258/real 1769075258] req@ffff9b0352a19500 x1855007617115264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075258.963861:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617115264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075258.963871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075258.963876:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075258.963880:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617115264:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075258.963886:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115392:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075258.963904:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075258.963911:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075258/real 1769075258] req@ffff9b0352a18000 x1855007617115392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075258.963919:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617115392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075258.963924:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075258.963926:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075258.963928:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115392:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075258.963988:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115520:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075258.964004:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075258.964012:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075258/real 1769075258] req@ffff9b0352a1b800 x1855007617115520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075258.964020:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617115520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075258.964030:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075258.964032:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075258.964034:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115520:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075258.964038:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115648:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075258.964049:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075258.964066:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075258/real 1769075258] req@ffff9b0352a18380 x1855007617115648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075258.964073:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617115648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075258.964078:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075258.964080:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075258.964081:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115648:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075258.964085:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115776:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075258.964094:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075258.964099:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075258/real 1769075258] req@ffff9b0352a19c00 x1855007617115776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075258.964106:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617115776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075258.964110:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075258.964113:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075258.964115:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617115776:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075259.475334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075259.859371:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075260.499373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075260.883430:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075261.523415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075261.907482:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075262.547391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075262.931433:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075263.571467:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075263.955394:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075264.083457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075264.083470:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075264.083476:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075264.083488:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2253 00000100:00080000:3.0:1769075264.083502:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075264.083509:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075264.083559:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617115904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075264.083588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075264.083592:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075264.083594:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075264.083597:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2253 00000100:00080000:3.0:1769075264.083600:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075264.083603:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075264.083613:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075264.083626:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617116032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075264.083636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075264.083639:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075264.083640:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075264.083643:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2253 00000100:00080000:3.0:1769075264.083645:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075264.083649:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075264.083655:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075264.083666:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617116160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075264.083677:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617115904:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075264.083679:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075264.083691:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075264.083701:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075264.083707:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2253 00000100:00080000:3.0:1769075264.083709:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075264.083711:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075264.083716:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075264.083728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617116288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075264.083735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075264.083738:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075264.083740:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075264.083744:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2253 00000100:00080000:3.0:1769075264.083747:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075264.083750:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075264.083753:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075264.083761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617116416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075264.083791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075264.083915:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075264/real 1769075264] req@ffff9b0352a18a80 x1855007617115904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075264.083942:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617115904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075264.083957:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075264.083963:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075264.083969:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617115904:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075264.084002:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116032:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075264.084034:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075264.084061:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075264/real 1769075264] req@ffff9b0352a1b100 x1855007617116032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075264.084077:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617116032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075264.084087:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075264.084092:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075264.084097:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116032:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075264.084107:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116160:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075264.084140:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075264.084161:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075264/real 1769075264] req@ffff9b0352a1ad80 x1855007617116160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075264.084193:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617116160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075264.084211:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075264.084217:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075264.084224:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116160:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075264.084295:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116288:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075264.084342:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075264.084370:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075264/real 1769075264] req@ffff9b0352a1b480 x1855007617116288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075264.084387:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617116288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075264.084398:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075264.084402:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075264.084406:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116288:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075264.084414:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116416:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075264.084439:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075264.084451:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075264/real 1769075264] req@ffff9b0352a1a680 x1855007617116416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075264.084472:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617116416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075264.084482:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075264.084489:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075264.084494:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116416:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075264.595333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075264.979466:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075265.619394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075266.003453:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075266.643360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075267.027355:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075267.667400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075268.051437:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075268.691325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075269.075397:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075269.203388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075269.203400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075269.203405:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075269.203417:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2258 00000100:00080000:0.0:1769075269.203428:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075269.203436:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075269.203493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617116544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075269.203518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075269.203522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075269.203524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075269.203529:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2258 00000100:00080000:0.0:1769075269.203534:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075269.203539:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075269.203563:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075269.203567:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617116544:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075269.203582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b800 x1855007617116672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075269.203597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075269.203601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075269.203603:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075269.203606:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2258 00000100:00080000:0.0:1769075269.203610:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075269.203614:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075269.203619:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075269.203629:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617116800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075269.203636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075269.203639:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075269.203640:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075269.203643:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2258 00000100:00080000:0.0:1769075269.203654:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075269.203656:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075269.203660:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075269.203661:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075269.203672:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617116928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075269.203681:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075269.203684:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075269.203686:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075269.203689:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2258 00000100:00080000:0.0:1769075269.203692:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075269.203695:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075269.203698:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075269.203707:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617117056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075269.203714:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075269/real 1769075269] req@ffff9b0365a3aa00 x1855007617116544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075269.203734:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617116544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075269.203744:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075269.203749:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075269.203752:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617116544:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075269.203797:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116672:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075269.203828:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075269.203847:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075269/real 1769075269] req@ffff9b0365a3b800 x1855007617116672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075269.203857:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b800 x1855007617116672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075269.203864:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075269.203867:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075269.203869:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116672:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075269.203873:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116800:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075269.203886:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075269.203892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075269/real 1769075269] req@ffff9b0365a39c00 x1855007617116800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075269.203900:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617116800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075269.203906:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075269.203909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075269.203911:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116800:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075269.203915:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116928:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075269.203926:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075269.203931:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075269/real 1769075269] req@ffff9b0365a39500 x1855007617116928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075269.203940:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617116928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075269.203945:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075269.203947:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075269.203949:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617116928:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075269.203952:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117056:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075269.203961:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075269.203967:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075269/real 1769075269] req@ffff9b0365a3bb80 x1855007617117056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075269.203975:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617117056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075269.203980:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075269.203982:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075269.203984:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117056:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075269.715312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075270.099398:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075270.739368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075271.123403:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075271.763368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075272.147434:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075272.787326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075273.171375:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075273.811409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075274.195401:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075274.323449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075274.323462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075274.323470:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075274.323487:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2263 00000100:00080000:0.0:1769075274.323513:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075274.323526:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075274.323580:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617117184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075274.323629:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075274.323634:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075274.323637:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075274.323642:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2263 00000100:00080000:0.0:1769075274.323647:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075274.323654:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075274.323669:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075274.323713:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617117312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075274.323729:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617117184:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075274.323733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075274.323744:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075274.323749:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075274.323756:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2263 00000100:00080000:0.0:1769075274.323763:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075274.323772:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075274.323784:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075274.323799:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617117440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075274.323829:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075274.323834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075274.323837:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075274.323842:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2263 00000100:00080000:0.0:1769075274.323847:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075274.323908:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075274.323917:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075274.323936:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617117568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075274.323948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075274.323953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075274.323956:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075274.323961:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2263 00000100:00080000:0.0:1769075274.323966:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075274.323970:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075274.323977:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075274.323990:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617117696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075274.324171:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075274.324427:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075274/real 1769075274] req@ffff9b0365a3b100 x1855007617117184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075274.324462:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617117184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075274.324479:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075274.324486:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075274.324492:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617117184:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075274.324530:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117312:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075274.324565:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075274.324582:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075274/real 1769075274] req@ffff9b0365a3ad80 x1855007617117312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075274.324598:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617117312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075274.324608:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075274.324613:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075274.324617:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117312:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075274.324625:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117440:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075274.324659:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075274.324718:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075274/real 1769075274] req@ffff9b0365a39180 x1855007617117440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075274.324752:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617117440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075274.324768:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075274.324774:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075274.324780:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117440:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075274.324802:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117568:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075274.324851:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075274.324867:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075274/real 1769075274] req@ffff9b0365a3bb80 x1855007617117568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075274.324894:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617117568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075274.324903:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075274.324907:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075274.324910:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117568:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075274.324917:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117696:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075274.324935:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075274.324944:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075274/real 1769075274] req@ffff9b0365a39500 x1855007617117696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075274.324957:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617117696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075274.324966:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075274.324969:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075274.324973:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117696:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075274.835400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075275.219464:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075275.859510:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075276.243327:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075276.883370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075277.267398:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075277.907324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075278.291395:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075278.931422:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075279.315550:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075279.443353:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075279.443362:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075279.443367:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075279.443376:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2268 00000100:00080000:3.0:1769075279.443384:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075279.443390:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075279.443425:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617117824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075279.443451:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075279.443454:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075279.443456:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075279.443459:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2268 00000100:00080000:3.0:1769075279.443462:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075279.443465:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075279.443473:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075279.443483:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617117952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075279.443492:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617117824:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075279.443496:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075279.443503:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075279.443508:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075279.443512:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2268 00000100:00080000:3.0:1769075279.443514:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075279.443516:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075279.443520:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075279.443531:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617118080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075279.443539:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075279.443542:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075279.443544:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075279.443548:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2268 00000100:00080000:3.0:1769075279.443559:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075279.443563:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075279.443567:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075279.443570:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075279.443581:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617118208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075279.443590:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075279.443614:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075279.443616:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075279.443620:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2268 00000100:00080000:3.0:1769075279.443625:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075279.443630:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075279.443634:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075279.443645:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075279/real 1769075279] req@ffff9b0352a19f80 x1855007617117824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075279.443651:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617118336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075279.443684:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617117824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075279.443696:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075279.443702:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075279.443705:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617117824:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075279.443726:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117952:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075279.443746:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075279.443758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075279/real 1769075279] req@ffff9b0352a1a300 x1855007617117952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075279.443773:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617117952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075279.443783:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075279.443787:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075279.443791:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617117952:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075279.443799:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118080:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075279.443824:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075279.443838:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075279/real 1769075279] req@ffff9b0352a18e00 x1855007617118080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075279.443861:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617118080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075279.443869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075279.443871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075279.443874:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118080:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075279.443879:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118208:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075279.443897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075279.443905:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075279/real 1769075279] req@ffff9b0352a18700 x1855007617118208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075279.443913:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617118208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075279.443919:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075279.443922:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075279.443924:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118208:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075279.443928:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118336:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075279.443939:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075279.443945:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075279/real 1769075279] req@ffff9b0352a1aa00 x1855007617118336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075279.443953:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617118336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075279.443963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075279.443966:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075279.443968:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118336:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075279.955311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075280.339348:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075280.979346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075281.363391:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075282.003408:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075282.387482:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075283.027421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075283.411408:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075284.051349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075284.435358:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075284.563324:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075284.563333:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075284.563337:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075284.563344:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2273 00000100:00080000:2.0:1769075284.563352:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075284.563357:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075284.563388:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7800 x1855007617118464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075284.563415:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075284.563418:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075284.563420:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075284.563423:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2273 00000100:00080000:2.0:1769075284.563425:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075284.563428:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075284.563435:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075284.563444:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617118592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075284.563451:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617118464:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075284.563455:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075284.563459:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075284.563460:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075284.563463:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2273 00000100:00080000:2.0:1769075284.563465:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075284.563467:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075284.563471:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075284.563478:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617118720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075284.563484:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075284.563486:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075284.563487:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075284.563490:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2273 00000100:00080000:2.0:1769075284.563499:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075284.563502:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075284.563505:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075284.563525:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617118848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075284.563531:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075284.563533:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075284.563534:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075284.563537:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2273 00000100:00080000:2.0:1769075284.563539:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075284.563543:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769075284.563559:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769075284.563561:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075284.563569:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617118976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075284.563605:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075284/real 1769075284] req@ffff9b03482c7800 x1855007617118464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075284.563621:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7800 x1855007617118464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075284.563629:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075284.563632:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075284.563635:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617118464:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075284.563654:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118592:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075284.563671:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075284.563680:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075284/real 1769075284] req@ffff9b03482c5180 x1855007617118592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075284.563688:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617118592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075284.563693:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075284.563696:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075284.563698:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118592:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075284.563702:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118720:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075284.563712:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075284.563717:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075284/real 1769075284] req@ffff9b03482c4700 x1855007617118720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075284.563724:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617118720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075284.563729:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075284.563731:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075284.563733:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118720:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075284.563736:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118848:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075284.563746:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075284.563752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075284/real 1769075284] req@ffff9b03482c7100 x1855007617118848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075284.563760:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617118848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075284.563765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075284.563766:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075284.563768:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118848:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075284.563772:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118976:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075284.563781:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075284.563786:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075284/real 1769075284] req@ffff9b03482c4000 x1855007617118976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075284.563793:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617118976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075284.563798:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075284.563800:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075284.563802:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617118976:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075285.075373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075285.459352:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075286.099374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075286.483570:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075287.123357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075287.507362:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075288.147387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075288.531482:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075289.171349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075289.555450:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075289.683470:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075289.683489:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075289.683495:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075289.683511:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2278 00000100:00080000:0.0:1769075289.683525:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075289.683534:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075289.683587:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617119104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075289.683625:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075289.683629:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075289.683631:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075289.683635:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2278 00000100:00080000:0.0:1769075289.683638:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075289.683642:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075289.683653:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075289.683668:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b800 x1855007617119232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075289.683670:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617119104:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075289.683692:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075289.683698:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075289.683701:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075289.683707:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2278 00000100:00080000:0.0:1769075289.683713:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075289.683719:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075289.683727:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075289.683743:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617119360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075289.683754:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075289.683757:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075289.683759:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769075289.683765:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075289.683768:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2278 00000100:00080000:0.0:1769075289.683775:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075289.683779:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075289.683786:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075289.683797:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617119488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075289.683806:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075289.683810:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075289.683812:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075289.683818:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2278 00000100:00100000:3.0:1769075289.683818:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075289/real 1769075289] req@ffff9b0365a39c00 x1855007617119104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075289.683821:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075289.683829:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075289.683838:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075289.683850:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617119616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075289.683853:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617119104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075289.683882:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075289.683887:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075289.683892:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617119104:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075289.683920:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617119232:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075289.683940:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075289.683950:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075289/real 1769075289] req@ffff9b0365a3b800 x1855007617119232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075289.683961:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b800 x1855007617119232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075289.683968:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075289.683971:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075289.683974:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617119232:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075289.683978:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617119360:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075289.683991:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075289.683998:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075289/real 1769075289] req@ffff9b0365a3aa00 x1855007617119360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075289.684007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617119360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075289.684013:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075289.684015:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075289.684017:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617119360:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075289.684022:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617119488:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075289.684035:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075289.684043:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075289/real 1769075289] req@ffff9b0365a3b480 x1855007617119488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075289.684052:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617119488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075289.684058:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075289.684061:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075289.684063:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617119488:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075289.684067:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617119616:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075289.684078:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075289.684084:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075289/real 1769075289] req@ffff9b0365a38e00 x1855007617119616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075289.684093:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617119616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075289.684101:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075289.684104:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075289.684108:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617119616:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075290.195465:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075290.579563:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075291.219372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075291.603406:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075292.243347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075292.627455:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075293.267403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075293.651495:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075294.291387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075294.675475:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075294.803378:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075294.803390:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075294.803396:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075294.803425:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2283 00000100:00080000:0.0:1769075294.803438:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075294.803446:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075294.803487:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39f80 x1855007617119744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075294.803515:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075294.803518:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075294.803520:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075294.803525:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2283 00000100:00080000:0.0:1769075294.803529:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075294.803533:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075294.803543:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075294.803556:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617119744:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075294.803557:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38000 x1855007617119872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075294.803577:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075294.803581:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075294.803610:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075294.803616:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2283 00000100:00080000:0.0:1769075294.803621:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075294.803626:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075294.803633:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075294.803667:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617120000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075294.803682:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075294.803684:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075294.803686:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075294.803689:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2283 00000800:00000100:3.0:1769075294.803694:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075294.803695:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075294.803704:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075294.803710:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075294.803722:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617120128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075294.803730:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075294.803733:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075294.803735:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075294.803739:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2283 00000100:00100000:3.0:1769075294.803745:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075294/real 1769075294] req@ffff9b0365a39f80 x1855007617119744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075294.803746:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075294.803755:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075294.803764:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075294.803778:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617120256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075294.803782:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39f80 x1855007617119744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075294.803811:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075294.803816:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075294.803821:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617119744:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075294.803843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617119872:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075294.803862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075294.803873:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075294/real 1769075294] req@ffff9b0365a38000 x1855007617119872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075294.803883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38000 x1855007617119872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075294.803890:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075294.803893:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075294.803896:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617119872:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075294.803900:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120000:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075294.803912:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075294.803919:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075294/real 1769075294] req@ffff9b0365a38a80 x1855007617120000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075294.803929:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617120000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075294.803941:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075294.803943:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075294.803946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120000:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075294.803951:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120128:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075294.803963:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075294.803970:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075294/real 1769075294] req@ffff9b0365a38e00 x1855007617120128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075294.803979:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617120128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075294.803985:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075294.803987:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075294.803990:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120128:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075294.803994:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120256:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075294.804005:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075294.804012:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075294/real 1769075294] req@ffff9b0365a3b480 x1855007617120256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075294.804020:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617120256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075294.804026:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075294.804028:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075294.804031:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120256:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075295.315403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075295.699404:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075296.339350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075296.723593:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075297.363352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075297.747447:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075298.387479:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075298.771523:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075299.411422:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075299.795494:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075299.923352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075299.923362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075299.923367:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075299.923376:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2289 00000100:00080000:2.0:1769075299.923385:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075299.923392:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075299.923427:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617120384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075299.923452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075299.923455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075299.923457:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075299.923461:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2289 00000100:00080000:2.0:1769075299.923463:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075299.923467:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075299.923475:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075299.923487:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617120512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075299.923496:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617120384:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075299.923511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075299.923515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075299.923518:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075299.923523:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2289 00000100:00080000:2.0:1769075299.923526:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075299.923529:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075299.923534:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075299.923543:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617120640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075299.923551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075299.923554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075299.923555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075299.923560:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2289 00000800:00000100:3.0:1769075299.923565:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075299.923567:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075299.923573:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075299.923578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075299.923588:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617120768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075299.923596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075299.923599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075299.923601:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075299.923605:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2289 00000100:00080000:2.0:1769075299.923608:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075299.923613:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075299/real 1769075299] req@ffff9b03482c6a00 x1855007617120384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075299.923615:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075299.923623:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075299.923637:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617120896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075299.923644:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617120384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075299.923660:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075299.923664:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075299.923668:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617120384:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075299.923691:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120512:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075299.923709:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075299.923718:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075299/real 1769075299] req@ffff9b03482c5f80 x1855007617120512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075299.923728:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617120512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075299.923735:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075299.923738:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075299.923740:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120512:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075299.923745:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120640:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075299.923756:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075299.923763:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075299/real 1769075299] req@ffff9b03482c4e00 x1855007617120640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075299.923773:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617120640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075299.923779:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075299.923781:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075299.923783:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120640:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075299.923788:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120768:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075299.923800:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075299.923806:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075299/real 1769075299] req@ffff9b03482c7b80 x1855007617120768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075299.923815:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617120768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075299.923821:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075299.923824:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075299.923826:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120768:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075299.923830:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120896:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075299.923840:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075299.923847:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075299/real 1769075299] req@ffff9b03482c5c00 x1855007617120896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075299.923855:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617120896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075299.923861:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075299.923864:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075299.923866:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617120896:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075300.435433:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075300.819384:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075301.459380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075301.843406:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075302.483359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075302.867341:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075303.507308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075303.891409:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075304.531365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075304.915368:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075305.043385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075305.043397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075305.043402:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075305.043440:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2294 00000100:00080000:3.0:1769075305.043452:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075305.043460:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075305.043504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617121024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075305.043538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075305.043541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075305.043543:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075305.043547:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2294 00000100:00080000:3.0:1769075305.043566:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075305.043569:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075305.043578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075305.043588:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617121152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075305.043598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075305.043602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075305.043604:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769075305.043613:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617121024:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075305.043613:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2294 00000100:00080000:3.0:1769075305.043618:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075305.043625:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075305.043640:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075305.043653:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617121280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075305.043659:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075305.043662:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075305.043664:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075305.043668:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2294 00000100:00080000:3.0:1769075305.043670:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075305.043673:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075305.043676:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075305.043686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617121408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075305.043704:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075305.043707:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075305.043709:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075305.043711:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2294 00000100:00080000:3.0:1769075305.043725:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075305.043729:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075305.043732:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075305.043742:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617121536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075305.043749:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075305.043805:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075305/real 1769075305] req@ffff9b0352a1aa00 x1855007617121024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075305.043840:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617121024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075305.043857:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075305.043863:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075305.043868:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617121024:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075305.043896:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121152:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075305.043923:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075305.043938:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075305/real 1769075305] req@ffff9b0352a18700 x1855007617121152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075305.043953:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617121152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075305.043963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075305.043968:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075305.043972:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121152:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075305.043979:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121280:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075305.044001:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075305.044013:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075305/real 1769075305] req@ffff9b0352a18e00 x1855007617121280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075305.044044:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617121280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075305.044053:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075305.044056:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075305.044060:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121280:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075305.044067:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121408:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075305.044088:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075305.044115:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075305/real 1769075305] req@ffff9b0352a1a300 x1855007617121408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075305.044130:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617121408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075305.044139:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075305.044143:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075305.044148:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121408:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075305.044156:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121536:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075305.044201:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075305.044222:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075305/real 1769075305] req@ffff9b0352a19f80 x1855007617121536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075305.044275:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617121536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075305.044285:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075305.044289:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075305.044293:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121536:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075305.555327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075305.939379:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075306.579405:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075306.963396:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075307.603337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075307.987390:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075308.627339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075309.011317:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075309.651349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075310.035402:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075310.163306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075310.163316:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075310.163320:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075310.163328:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2299 00000100:00080000:3.0:1769075310.163337:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075310.163344:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075310.163375:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617121664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075310.163393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075310.163396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075310.163398:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075310.163403:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2299 00000100:00080000:3.0:1769075310.163406:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075310.163409:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075310.163417:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075310.163418:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617121664:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075310.163430:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617121792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075310.163439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075310.163442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075310.163444:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075310.163447:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2299 00000100:00080000:3.0:1769075310.163450:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075310.163452:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075310.163455:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075310.163476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617121920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075310.163491:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075310.163491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075310.163499:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075310.163501:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075310.163505:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2299 00000100:00080000:3.0:1769075310.163509:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075310.163512:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075310.163516:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075310.163525:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617122048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075310.163532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075310.163535:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075310.163537:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075310.163541:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2299 00000100:00100000:2.0:1769075310.163543:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075310/real 1769075310] req@ffff9b0352a1a680 x1855007617121664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075310.163545:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075310.163551:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075310.163556:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075310.163570:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617122176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075310.163579:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617121664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075310.163595:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075310.163603:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075310.163609:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617121664:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075310.163632:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121792:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075310.163657:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075310.163669:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075310/real 1769075310] req@ffff9b0352a1b480 x1855007617121792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075310.163682:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617121792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075310.163697:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075310.163701:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075310.163705:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121792:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075310.163712:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121920:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075310.163733:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075310.163742:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075310/real 1769075310] req@ffff9b0352a1ad80 x1855007617121920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075310.163750:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617121920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075310.163755:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075310.163758:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075310.163760:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617121920:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075310.163764:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122048:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075310.163776:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075310.163786:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075310/real 1769075310] req@ffff9b0352a1b100 x1855007617122048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075310.163800:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617122048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075310.163809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075310.163813:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075310.163817:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122048:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075310.163824:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122176:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075310.163848:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075310.163857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075310/real 1769075310] req@ffff9b0352a18a80 x1855007617122176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075310.163866:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617122176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075310.163871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075310.163873:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075310.163875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122176:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075310.675386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075311.059457:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075311.699345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075312.083351:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075312.723329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075313.107328:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075313.747331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075314.131372:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075314.771329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075315.155397:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075315.283316:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075315.283325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075315.283334:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075315.283345:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2304 00000100:00080000:3.0:1769075315.283352:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075315.283358:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075315.283388:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617122304/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075315.283409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075315.283412:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075315.283413:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075315.283416:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2304 00000100:00080000:3.0:1769075315.283418:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075315.283420:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075315.283427:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075315.283434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617122432/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075315.283440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075315.283442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075315.283444:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075315.283446:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2304 00000100:00080000:3.0:1769075315.283448:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075315.283451:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075315.283454:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075315.283459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617122560/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075315.283465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075315.283467:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075315.283468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075315.283471:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2304 00000100:00080000:3.0:1769075315.283473:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075315.283476:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075315.283479:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075315.283484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617122688/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075315.283491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075315.283495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769075315.283496:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617122304:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075315.283497:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075315.283504:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2304 00000100:00080000:3.0:1769075315.283506:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075315.283508:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075315.283511:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075315.283518:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617122816/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075315.283588:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122432:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075315.283607:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122560:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075315.283628:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122688:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075315.283643:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122816:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075315.795366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075316.179345:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075316.819303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075317.203333:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075317.843338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075318.227457:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075318.355494:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769075318.355534:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075318.355553:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075318.355591:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075318.355599:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075318.355607:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075318.355618:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769075318.355667:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075315/real 1769075318] req@ffff9b0352a18000 x1855007617122688/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075318.355710:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617122688/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075318.355726:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075318.355733:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075318.355738:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122688:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075318.355749:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075315/real 1769075318] req@ffff9b0352a19500 x1855007617122816/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075318.355764:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617122816/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075318.355771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075318.355776:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075318.355779:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122816:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075318.355787:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075315/real 1769075318] req@ffff9b0352a19c00 x1855007617122304/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075318.355800:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617122304/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075318.355815:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075318.355819:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075318.355823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617122304:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075318.355831:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075315/real 1769075318] req@ffff9b0352a18380 x1855007617122432/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075318.355845:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617122432/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075318.355854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075318.355859:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075318.355862:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122432:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075318.355872:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075315/real 1769075318] req@ffff9b0352a1b800 x1855007617122560/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075318.355885:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617122560/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075318.355894:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075318.355898:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075318.355902:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617122560:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075318.867331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075319.251360:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075319.891815:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075320.275354:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075320.403364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075320.403378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075320.403385:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075320.403398:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2309 00000100:00080000:0.0:1769075320.403427:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075320.403446:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075320.403487:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617122944/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075320.403514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075320.403518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075320.403519:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075320.403523:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2309 00000100:00080000:0.0:1769075320.403527:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075320.403533:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075320.403554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075320.403565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b800 x1855007617123072/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075320.403576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075320.403581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075320.403587:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769075320.403591:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617122944:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075320.403598:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2309 00000100:00080000:0.0:1769075320.403609:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075320.403626:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075320.403633:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075320.403652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617123200/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075320.403665:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075320.403669:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075320.403672:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075320.403684:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2309 00000100:00080000:0.0:1769075320.403692:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075320.403697:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075320.403706:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075320.403707:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075320.403731:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617123328/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075320.403747:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075320.403751:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075320.403755:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075320.403762:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2309 00000100:00100000:2.0:1769075320.403769:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075320/real 1769075320] req@ffff9b0365a3aa00 x1855007617122944/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075320.403773:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075320.403786:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075320.403800:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075320.403814:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617122944/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075320.403822:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617123456/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075320.403849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075320.403862:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075320.403870:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617122944:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075320.403905:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123072:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075320.403976:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075320.404000:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075320/real 1769075320] req@ffff9b0365a3b800 x1855007617123072/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075320.404015:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b800 x1855007617123072/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075320.404024:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075320.404027:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075320.404031:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123072:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075320.404038:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123200:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075320.404061:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075320.404073:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075320/real 1769075320] req@ffff9b0365a39c00 x1855007617123200/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075320.404085:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617123200/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075320.404092:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075320.404096:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075320.404100:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123200:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075320.404106:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123328:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075320.404128:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075320.404139:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075320/real 1769075320] req@ffff9b0365a39500 x1855007617123328/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075320.404167:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617123328/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075320.404175:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075320.404179:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075320.404182:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123328:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075320.404189:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123456:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075320.404210:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075320.404222:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075320/real 1769075320] req@ffff9b0365a3bb80 x1855007617123456/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075320.404271:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617123456/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075320.404280:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075320.404283:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075320.404287:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123456:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075320.915325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075321.299343:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075321.939366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075322.323394:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075322.963360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075323.347391:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075323.987350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075324.371345:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075325.011365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075325.395476:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075325.523358:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075325.523371:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075325.523377:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075325.523388:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2314 00000100:00080000:0.0:1769075325.523399:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075325.523407:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075325.523453:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617123584/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075325.523484:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075325.523489:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075325.523491:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075325.523497:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2314 00000100:00080000:0.0:1769075325.523501:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075325.523506:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075325.523519:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075325.523532:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617123584:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075325.523538:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617123712/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075325.523549:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075325.523552:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075325.523553:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075325.523558:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2314 00000100:00080000:0.0:1769075325.523561:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075325.523564:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075325.523568:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075325.523576:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617123840/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075325.523582:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075325.523585:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075325.523586:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075325.523588:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2314 00000100:00080000:0.0:1769075325.523590:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075325.523593:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075325.523596:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075325.523603:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617123968/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075325.523608:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075325.523611:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075325.523612:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075325.523616:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2314 00000100:00080000:0.0:1769075325.523619:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075325.523621:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075325.523624:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075325.523633:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617124096/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075325.523635:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075325.523697:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075325/real 1769075325] req@ffff9b0365a39180 x1855007617123584/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075325.523720:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617123584/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075325.523732:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075325.523738:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075325.523742:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617123584:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075325.523772:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123712:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075325.523799:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075325.523813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075325/real 1769075325] req@ffff9b0365a3ad80 x1855007617123712/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075325.523844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617123712/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075325.523854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075325.523858:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075325.523862:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123712:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075325.523869:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123840:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075325.523891:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075325.523903:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075325/real 1769075325] req@ffff9b0365a3b100 x1855007617123840/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075325.523916:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617123840/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075325.523932:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075325.523945:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075325.523949:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123840:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075325.523956:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123968:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075325.523978:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075325.523990:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075325/real 1769075325] req@ffff9b0365a3bb80 x1855007617123968/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075325.524002:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617123968/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075325.524010:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075325.524014:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075325.524017:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617123968:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075325.524024:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124096:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075325.524044:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075325.524055:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075325/real 1769075325] req@ffff9b0365a39500 x1855007617124096/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075325.524067:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617124096/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075325.524076:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075325.524079:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075325.524084:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124096:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075326.035347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075326.419354:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075327.059380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075327.443369:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075328.083385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075328.467385:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075329.107346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075329.491471:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075330.131382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075330.515365:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075330.643398:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075330.643409:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075330.643413:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075330.643421:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2319 00000100:00080000:1.0:1769075330.643431:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075330.643437:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075330.643468:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617124224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075330.643492:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075330.643495:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075330.643496:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075330.643499:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2319 00000100:00080000:1.0:1769075330.643501:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075330.643504:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075330.643512:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075330.643522:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617124352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075330.643530:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075330.643533:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075330.643536:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075330.643544:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2319 00000100:00100000:2.0:1769075330.643544:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617124224:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075330.643554:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075330.643562:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075330.643565:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075330.643575:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617124480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075330.643581:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075330.643584:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075330.643586:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075330.643589:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2319 00000100:00080000:1.0:1769075330.643592:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075330.643594:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075330.643597:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075330.643606:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617124608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075330.643612:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075330.643614:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075330.643615:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075330.643618:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2319 00000100:00080000:1.0:1769075330.643628:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075330.643630:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075330.643635:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075330.643643:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075330.643648:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617124736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075330.643700:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075330/real 1769075330] req@ffff9b0343dedf80 x1855007617124224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075330.643717:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617124224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075330.643726:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075330.643730:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075330.643757:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617124224:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075330.643786:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124352:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075330.643807:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075330.643816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075330/real 1769075330] req@ffff9b0343dec700 x1855007617124352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075330.643824:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617124352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075330.643831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075330.643833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075330.643836:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124352:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075330.643841:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124480:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075330.643853:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075330.643859:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075330/real 1769075330] req@ffff9b0343dec000 x1855007617124480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075330.643866:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617124480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075330.643871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075330.643873:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075330.643875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124480:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075330.643879:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124608:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075330.643890:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075330.643895:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075330/real 1769075330] req@ffff9b0343ded180 x1855007617124608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075330.643902:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617124608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075330.643907:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075330.643909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075330.643911:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124608:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075330.643915:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124736:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075330.643925:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075330.643931:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075330/real 1769075330] req@ffff9b0343dec380 x1855007617124736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075330.643939:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617124736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075330.643944:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075330.643946:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075330.643948:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124736:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075331.155345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075331.539334:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075332.179333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075332.563427:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075333.203284:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075333.587368:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075334.227424:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075334.611431:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075335.251381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075335.635397:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075335.763338:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075335.763349:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075335.763354:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075335.763365:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2324 00000100:00080000:3.0:1769075335.763376:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075335.763383:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075335.763426:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617124864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075335.763452:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075335.763455:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075335.763456:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075335.763459:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2324 00000100:00080000:3.0:1769075335.763462:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075335.763464:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075335.763473:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075335.763482:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617124992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075335.763489:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075335.763492:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075335.763493:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075335.763496:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2324 00000100:00080000:3.0:1769075335.763498:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075335.763500:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075335.763505:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075335.763511:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617125120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075335.763537:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075335.763540:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075335.763541:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075335.763544:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2324 00000100:00080000:3.0:1769075335.763547:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075335.763550:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075335.763557:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075335.763564:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617125248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075335.763571:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075335.763574:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075335.763575:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075335.763578:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2324 00000100:00080000:3.0:1769075335.763580:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075335.763582:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075335.763586:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075335.763594:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617125376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075335.763608:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617124864:192.168.204.160@tcp:250:kworker.0 00000800:00000100:2.0:1769075335.763727:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075335.763768:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075335/real 1769075335] req@ffff9b0352a1b800 x1855007617124864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075335.763785:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617124864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075335.763793:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075335.763797:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075335.763801:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617124864:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075335.763806:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124992:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075335.763824:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075335.763832:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075335/real 1769075335] req@ffff9b0352a18380 x1855007617124992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075335.763841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617124992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075335.763846:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075335.763848:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075335.763851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617124992:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075335.763855:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125120:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075335.763866:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075335.763872:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075335/real 1769075335] req@ffff9b0352a19c00 x1855007617125120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075335.763880:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617125120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075335.763885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075335.763887:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075335.763889:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125120:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075335.763893:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125248:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075335.763904:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075335.763910:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075335/real 1769075335] req@ffff9b0352a19500 x1855007617125248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075335.763923:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617125248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075335.763928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075335.763930:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075335.763932:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125248:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075335.763956:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125376:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075335.763967:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075335.763974:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075335/real 1769075335] req@ffff9b0352a18000 x1855007617125376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075335.763981:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617125376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075335.763987:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075335.763989:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075335.763991:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125376:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075336.275313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075336.659387:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075337.299347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075337.683355:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075338.323370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075338.707403:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075339.347356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075339.731427:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075340.371319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075340.755392:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075340.883387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075340.883398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075340.883403:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075340.883412:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2330 00000100:00080000:1.0:1769075340.883420:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075340.883426:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075340.883458:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617125504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075340.883481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075340.883484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075340.883485:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075340.883488:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2330 00000100:00080000:1.0:1769075340.883490:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075340.883492:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075340.883499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075340.883510:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617125632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075340.883517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075340.883520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075340.883521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075340.883523:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2330 00000100:00080000:1.0:1769075340.883526:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075340.883530:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617125504:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075340.883531:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075340.883538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075340.883546:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617125760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075340.883557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075340.883560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075340.883561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075340.883583:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2330 00000100:00080000:1.0:1769075340.883586:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075340.883588:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075340.883592:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075340.883601:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedc00 x1855007617125888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075340.883608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075340.883610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075340.883611:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075340.883614:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2330 00000100:00080000:1.0:1769075340.883623:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075340.883626:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075340.883629:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075340.883631:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075340.883639:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617126016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075340.883669:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075340/real 1769075340] req@ffff9b0343dee300 x1855007617125504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075340.883682:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617125504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075340.883690:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075340.883693:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075340.883696:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617125504:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075340.883701:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125632:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075340.883715:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075340.883722:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075340/real 1769075340] req@ffff9b0343def100 x1855007617125632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075340.883730:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617125632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075340.883736:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075340.883737:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075340.883739:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125632:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075340.883758:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125760:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075340.883769:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075340.883774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075340/real 1769075340] req@ffff9b0343ded880 x1855007617125760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075340.883782:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617125760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075340.883792:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075340.883794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075340.883796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125760:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075340.883799:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125888:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075340.883808:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075340.883813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075340/real 1769075340] req@ffff9b0343dedc00 x1855007617125888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075340.883820:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedc00 x1855007617125888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075340.883825:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075340.883827:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075340.883829:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617125888:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075340.883832:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126016:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075340.883840:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075340.883845:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075340/real 1769075340] req@ffff9b0343def800 x1855007617126016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075340.883852:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617126016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075340.883856:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075340.883858:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075340.883860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126016:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075341.395358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075341.779429:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075342.419329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075342.803408:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075343.443325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075343.827407:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075344.467317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075344.851367:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075345.491358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075345.875412:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075346.003377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075346.003389:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075346.003394:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075346.003405:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2335 00000100:00080000:0.0:1769075346.003418:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075346.003427:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075346.003471:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617126144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075346.003517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075346.003521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075346.003526:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075346.003534:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2335 00000100:00080000:0.0:1769075346.003540:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075346.003545:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075346.003559:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075346.003570:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617126144:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075346.003578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b800 x1855007617126272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075346.003591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075346.003594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075346.003596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075346.003599:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2335 00000100:00080000:0.0:1769075346.003603:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075346.003608:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075346.003612:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075346.003624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617126400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075346.003633:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075346.003643:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1769075346.003645:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075346.003648:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075346.003659:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2335 00000100:00080000:0.0:1769075346.003663:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075346.003668:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075346.003675:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075346.003684:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617126528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075346.003691:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff9b0365a39c00 x1855007617126144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075346.003755:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075346.003764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075346.003768:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075346.003773:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2335 00000100:00080000:0.0:1769075346.003780:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075346.003785:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075346.003790:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075346.003798:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617126656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075346.003829:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617126144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075346.003844:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075346.003849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075346.003854:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617126144:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075346.003883:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126272:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075346.003916:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075346.003935:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff9b0365a3b800 x1855007617126272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075346.003963:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b800 x1855007617126272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075346.003976:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075346.003981:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075346.003988:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126272:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075346.003999:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126400:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075346.004030:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075346.004051:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff9b0365a3aa00 x1855007617126400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075346.004067:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617126400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075346.004078:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075346.004081:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075346.004084:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126400:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075346.004090:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126528:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075346.004106:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075346.004112:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff9b0365a3b480 x1855007617126528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075346.004120:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617126528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075346.004125:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075346.004127:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075346.004130:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126528:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075346.004134:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126656:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075346.004143:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075346.004149:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075346/real 1769075346] req@ffff9b0365a38e00 x1855007617126656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075346.004156:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617126656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075346.004160:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075346.004162:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075346.004165:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126656:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075346.515324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075346.899421:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075347.539348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075347.923342:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075348.563315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075348.947346:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075349.587339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075349.971334:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075350.611304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075350.995447:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075351.123324:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075351.123333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075351.123337:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075351.123345:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2340 00000100:00080000:0.0:1769075351.123353:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075351.123358:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075351.123389:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617126784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075351.123431:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075351.123434:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075351.123435:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075351.123440:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2340 00000100:00080000:0.0:1769075351.123443:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075351.123446:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075351.123454:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075351.123464:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38000 x1855007617126912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075351.123471:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617126784:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075351.123475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075351.123479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075351.123480:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075351.123484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2340 00000100:00080000:0.0:1769075351.123486:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075351.123489:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075351.123492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075351.123499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39f80 x1855007617127040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075351.123505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075351.123508:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075351.123509:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075351.123512:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2340 00000100:00080000:0.0:1769075351.123521:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075351.123523:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075351.123527:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075351.123531:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075351.123539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617127168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075351.123547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075351.123550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075351.123551:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075351.123554:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2340 00000100:00080000:0.0:1769075351.123556:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075351.123559:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075351.123563:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075351.123567:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075351/real 1769075351] req@ffff9b0365a38a80 x1855007617126784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075351.123573:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617127296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075351.123594:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617126784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075351.123604:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075351.123607:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075351.123610:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617126784:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075351.123629:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126912:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075351.123643:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075351.123651:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075351/real 1769075351] req@ffff9b0365a38000 x1855007617126912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075351.123659:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38000 x1855007617126912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075351.123664:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075351.123666:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075351.123669:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617126912:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075351.123672:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127040:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075351.123681:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075351.123687:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075351/real 1769075351] req@ffff9b0365a39f80 x1855007617127040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075351.123694:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39f80 x1855007617127040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075351.123698:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075351.123700:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075351.123702:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127040:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075351.123705:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127168:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075351.123715:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075351.123720:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075351/real 1769075351] req@ffff9b0365a38e00 x1855007617127168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075351.123727:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617127168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075351.123732:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075351.123733:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075351.123735:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127168:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075351.123739:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127296:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075351.123747:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075351.123752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075351/real 1769075351] req@ffff9b0365a3b480 x1855007617127296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075351.123758:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617127296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075351.123763:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075351.123765:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075351.123767:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127296:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075351.635338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075352.019473:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075352.659355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075353.043428:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075353.683337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075354.067449:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075354.707345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075355.091320:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075355.731344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075356.115381:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075356.243350:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075356.243360:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075356.243364:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075356.243377:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2345 00000100:00080000:0.0:1769075356.243385:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075356.243390:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075356.243423:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617127424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075356.243446:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075356.243449:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075356.243450:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075356.243454:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2345 00000100:00080000:0.0:1769075356.243457:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075356.243460:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075356.243468:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075356.243479:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617127424:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075356.243480:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b800 x1855007617127552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075356.243493:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075356.243496:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075356.243497:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075356.243499:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2345 00000100:00080000:0.0:1769075356.243502:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075356.243505:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075356.243508:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075356.243516:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617127680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075356.243523:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075356.243525:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075356.243527:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075356.243529:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2345 00000100:00080000:0.0:1769075356.243534:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075356.243536:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075356.243539:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075356.243545:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075356.243549:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617127808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075356.243556:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075356.243559:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075356.243560:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075356.243563:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2345 00000100:00080000:0.0:1769075356.243567:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075356.243569:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075356.243572:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075356.243579:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617127936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075356.243595:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075356/real 1769075356] req@ffff9b0365a3aa00 x1855007617127424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075356.243617:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617127424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075356.243625:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075356.243628:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075356.243631:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617127424:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075356.243649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127552:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075356.243667:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075356.243675:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075356/real 1769075356] req@ffff9b0365a3b800 x1855007617127552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075356.243683:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b800 x1855007617127552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075356.243689:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075356.243691:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075356.243693:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127552:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075356.243698:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127680:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075356.243707:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075356.243713:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075356/real 1769075356] req@ffff9b0365a39c00 x1855007617127680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075356.243720:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617127680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075356.243725:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075356.243727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075356.243729:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127680:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075356.243732:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127808:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075356.243742:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075356.243747:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075356/real 1769075356] req@ffff9b0365a39500 x1855007617127808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075356.243754:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617127808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075356.243759:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075356.243760:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075356.243762:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127808:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075356.243766:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127936:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075356.243774:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075356.243779:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075356/real 1769075356] req@ffff9b0365a3bb80 x1855007617127936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075356.243790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617127936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075356.243798:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075356.243801:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075356.243804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617127936:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075356.755349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075357.139433:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075357.779350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075358.163378:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075358.803347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075359.187419:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075359.827334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075360.211321:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075360.851337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075361.235432:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075361.363337:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075361.363349:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075361.363352:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075361.363360:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2350 00000100:00080000:3.0:1769075361.363368:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075361.363374:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075361.363405:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617128064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075361.363427:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075361.363429:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075361.363431:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075361.363441:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2350 00000100:00080000:3.0:1769075361.363444:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075361.363452:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075361.363454:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617128064:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769075361.363461:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075361.363472:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617128192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075361.363481:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075361.363484:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075361.363485:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075361.363488:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2350 00000100:00080000:3.0:1769075361.363491:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075361.363494:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075361.363497:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075361.363506:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617128320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075361.363513:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075361.363517:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075361.363520:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075361.363522:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075361.363525:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2350 00000100:00080000:3.0:1769075361.363528:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075361.363531:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075361.363534:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075361.363541:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617128448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075361.363548:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075361.363552:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075361/real 1769075361] req@ffff9b0352a18a80 x1855007617128064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075361.363553:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075361.363557:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075361.363564:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2350 00000100:00080000:3.0:1769075361.363569:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075361.363571:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075361.363575:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075361.363577:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617128064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075361.363585:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617128576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075361.363588:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075361.363596:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075361.363602:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617128064:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075361.363623:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128192:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075361.363638:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075361.363646:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075361/real 1769075361] req@ffff9b0352a1b100 x1855007617128192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075361.363653:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617128192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075361.363664:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075361.363666:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075361.363668:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128192:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075361.363672:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128320:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075361.363682:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075361.363687:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075361/real 1769075361] req@ffff9b0352a1ad80 x1855007617128320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075361.363694:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617128320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075361.363699:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075361.363701:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075361.363702:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128320:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075361.363706:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128448:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075361.363715:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075361.363720:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075361/real 1769075361] req@ffff9b0352a1b480 x1855007617128448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075361.363727:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617128448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075361.363732:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075361.363734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075361.363736:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128448:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075361.363740:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128576:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075361.363748:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075361.363753:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075361/real 1769075361] req@ffff9b0352a1a680 x1855007617128576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075361.363760:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617128576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075361.363765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075361.363767:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075361.363768:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128576:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075361.875325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075362.259562:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075362.899348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075363.283607:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075363.923314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075364.307698:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075364.947336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075365.331454:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075365.971330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075366.355699:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075366.483333:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075366.483342:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075366.483346:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075366.483355:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2355 00000100:00080000:3.0:1769075366.483364:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075366.483370:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075366.483399:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617128704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075366.483422:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075366.483425:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075366.483426:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075366.483429:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2355 00000100:00080000:3.0:1769075366.483432:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075366.483436:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075366.483443:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075366.483453:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617128704:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075366.483454:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617128832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075366.483468:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075366.483470:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075366.483472:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075366.483475:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2355 00000100:00080000:3.0:1769075366.483477:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075366.483480:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075366.483483:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075366.483493:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617128960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075366.483500:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075366.483502:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075366.483503:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075366.483505:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2355 00000100:00080000:3.0:1769075366.483507:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075366.483510:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075366.483514:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075366.483520:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617129088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075366.483526:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075366.483532:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075366.483535:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075366.483537:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075366.483540:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2355 00000100:00080000:3.0:1769075366.483543:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075366.483545:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075366.483548:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075366.483559:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617129216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075366.483574:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075366/real 1769075366] req@ffff9b0352a19f80 x1855007617128704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075366.483592:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617128704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075366.483600:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075366.483604:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075366.483608:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617128704:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075366.483629:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128832:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075366.483646:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075366.483655:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075366/real 1769075366] req@ffff9b0352a1a300 x1855007617128832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075366.483663:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617128832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075366.483669:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075366.483671:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075366.483673:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128832:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075366.483676:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128960:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075366.483687:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075366.483692:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075366/real 1769075366] req@ffff9b0352a18e00 x1855007617128960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075366.483699:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617128960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075366.483704:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075366.483706:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075366.483708:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617128960:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075366.483711:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129088:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075366.483720:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075366.483725:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075366/real 1769075366] req@ffff9b0352a18700 x1855007617129088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075366.483733:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617129088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075366.483738:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075366.483740:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075366.483741:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129088:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075366.483745:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129216:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075366.483753:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075366.483758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075366/real 1769075366] req@ffff9b0352a1aa00 x1855007617129216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075366.483765:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617129216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075366.483770:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075366.483772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075366.483774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129216:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075366.995321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075367.379418:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075368.019304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075368.403435:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075369.043363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075369.427357:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075370.067310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075370.451423:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075371.091321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075371.475417:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075371.603356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075371.603367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075371.603371:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075371.603381:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2360 00000100:00080000:2.0:1769075371.603390:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075371.603397:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075371.603433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617129344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075371.603463:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075371.603467:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075371.603469:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075371.603474:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2360 00000100:00080000:2.0:1769075371.603478:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075371.603483:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075371.603492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075371.603505:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617129344:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075371.603506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617129472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075371.603522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075371.603526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075371.603528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075371.603532:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2360 00000100:00080000:2.0:1769075371.603537:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075371.603542:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075371.603547:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075371.603559:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617129600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075371.603568:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075371.603572:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075371.603576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075371.603578:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075371.603584:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2360 00000100:00080000:2.0:1769075371.603588:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075371.603592:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075371.603598:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075371.603603:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075371/real 1769075371] req@ffff9b03482c6680 x1855007617129344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075371.603614:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617129728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075371.603621:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617129344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075371.603629:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075371.603636:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075371.603636:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075371.603640:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075371.603643:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075371.603650:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617129344:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075371.603650:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2360 00000100:00080000:2.0:1769075371.603659:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075371.603665:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075371.603670:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075371.603675:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129472:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075371.603682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617129856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075371.603696:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075371.603716:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075371/real 1769075371] req@ffff9b03482c5880 x1855007617129472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075371.603726:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617129472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075371.603731:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075371.603733:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075371.603735:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129472:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075371.603740:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129600:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075371.603751:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075371.603757:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075371/real 1769075371] req@ffff9b03482c5c00 x1855007617129600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075371.603764:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617129600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075371.603769:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075371.603771:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075371.603773:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129600:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075371.603778:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129728:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075371.603788:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075371.603794:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075371/real 1769075371] req@ffff9b03482c7b80 x1855007617129728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075371.603801:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617129728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075371.603806:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075371.603808:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075371.603809:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129728:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075371.603819:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129856:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075371.603828:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075371.603834:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075371/real 1769075371] req@ffff9b03482c4e00 x1855007617129856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075371.603841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617129856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075371.603846:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075371.603848:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075371.603851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617129856:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075372.115329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075372.499385:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075373.139332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075373.523489:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075374.163344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075374.547407:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075375.187400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075375.571492:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075376.211404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075376.595372:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075376.723346:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075376.723358:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075376.723362:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075376.723370:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2365 00000100:00080000:0.0:1769075376.723379:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075376.723385:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075376.723417:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617129984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075376.723444:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075376.723447:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075376.723448:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075376.723451:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2365 00000100:00080000:0.0:1769075376.723453:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075376.723456:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075376.723463:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075376.723472:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617130112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075376.723480:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075376.723483:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075376.723484:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075376.723488:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617129984:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075376.723489:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2365 00000100:00080000:0.0:1769075376.723496:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075376.723500:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075376.723503:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075376.723511:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617130240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075376.723517:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075376.723520:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075376.723527:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075376.723530:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2365 00000100:00080000:0.0:1769075376.723542:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075376.723545:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075376.723549:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075376.723553:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075376.723562:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617130368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075376.723570:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075376.723572:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075376.723574:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075376.723576:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2365 00000100:00080000:0.0:1769075376.723579:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075376.723582:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075376.723586:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075376.723591:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075376/real 1769075376] req@ffff9b0365a3b100 x1855007617129984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075376.723600:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617130496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075376.723611:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617129984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075376.723621:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075376.723624:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075376.723628:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617129984:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075376.723646:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130112:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075376.723660:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075376.723668:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075376/real 1769075376] req@ffff9b0365a3ad80 x1855007617130112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075376.723676:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617130112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075376.723682:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075376.723684:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075376.723686:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130112:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075376.723689:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130240:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075376.723698:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075376.723704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075376/real 1769075376] req@ffff9b0365a39180 x1855007617130240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075376.723711:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617130240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075376.723721:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075376.723723:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075376.723725:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130240:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075376.723728:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130368:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075376.723737:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075376.723742:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075376/real 1769075376] req@ffff9b0365a3bb80 x1855007617130368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075376.723749:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617130368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075376.723753:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075376.723755:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075376.723757:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130368:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075376.723761:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130496:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075376.723769:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075376.723784:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075376/real 1769075376] req@ffff9b0365a39500 x1855007617130496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075376.723795:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617130496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075376.723799:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075376.723801:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075376.723804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130496:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075377.235328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075377.619368:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075378.259335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075378.643379:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075379.283324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075379.667369:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075380.307344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075380.691385:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075381.331301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075381.715339:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075381.843407:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075381.843419:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075381.843425:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075381.843439:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2371 00000100:00080000:0.0:1769075381.843453:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075381.843462:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075381.843529:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617130624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075381.843570:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075381.843576:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075381.843579:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075381.843586:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2371 00000100:00080000:0.0:1769075381.843592:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075381.843597:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075381.843611:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075381.843612:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617130624:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075381.843636:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b800 x1855007617130752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075381.843651:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075381.843654:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075381.843657:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075381.843664:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2371 00000100:00080000:0.0:1769075381.843667:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075381.843672:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075381.843689:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075381.843703:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617130880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075381.843714:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075381.843718:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075381.843721:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075381.843725:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2371 00000100:00080000:0.0:1769075381.843732:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075381.843737:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075381.843741:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075381.843752:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617131008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075381.843760:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075381.843763:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075381.843764:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075381.843767:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2371 00000100:00080000:0.0:1769075381.843769:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075381.843773:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075381.843776:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075381.843791:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617131136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075381.844010:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130752:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075381.844057:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130880:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075381.844106:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131008:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075381.844146:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131136:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075382.355337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075382.739386:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075383.379301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075383.763375:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075384.403346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075384.787331:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075384.915671:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769075384.915766:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075384.915794:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075384.915909:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075384.915922:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075384.915931:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075384.915939:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:3.0:1769075384.916014:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075381/real 1769075384] req@ffff9b0365a3b800 x1855007617130752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075384.916057:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b800 x1855007617130752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075384.916071:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075384.916077:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075384.916082:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130752:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075384.916091:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075381/real 1769075384] req@ffff9b0365a3aa00 x1855007617130880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075384.916105:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617130880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075384.916113:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075384.916116:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075384.916119:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617130880:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075384.916125:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075381/real 1769075384] req@ffff9b0365a3b480 x1855007617131008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075384.916155:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617131008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075384.916167:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075384.916172:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075384.916177:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131008:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075384.916188:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075381/real 1769075384] req@ffff9b0365a38e00 x1855007617131136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075384.916209:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617131136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075384.916221:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075384.916226:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075384.916265:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131136:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075384.916277:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075381/real 1769075384] req@ffff9b0365a39c00 x1855007617130624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075384.916299:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617130624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075384.916312:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075384.916320:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075384.916328:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617130624:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075385.427333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075385.811419:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075386.451450:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075386.835381:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075386.963322:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075386.963331:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075386.963335:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075386.963344:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2376 00000100:00080000:0.0:1769075386.963353:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075386.963360:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075386.963400:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39f80 x1855007617131264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075386.963431:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075386.963435:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075386.963436:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075386.963440:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2376 00000100:00080000:0.0:1769075386.963445:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075386.963450:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075386.963478:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075386.963499:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617131264:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075386.963509:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38000 x1855007617131392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075386.963544:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075386.963548:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075386.963550:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075386.963553:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2376 00000100:00080000:0.0:1769075386.963557:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075386.963562:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075386.963568:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075386.963581:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617131520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075386.963593:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769075386.963599:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075386.963600:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075386.963604:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075386.963610:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2376 00000100:00080000:0.0:1769075386.963615:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075386.963619:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075386.963624:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075386.963634:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617131648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075386.963644:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075386.963648:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075386/real 1769075386] req@ffff9b0365a39f80 x1855007617131264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075386.963649:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075386.963651:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075386.963656:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2376 00000100:00080000:0.0:1769075386.963659:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075386.963667:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075386.963677:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075386.963691:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617131776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075386.963695:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39f80 x1855007617131264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075386.963721:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075386.963727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075386.963732:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617131264:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075386.963761:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131392:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075386.963807:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075386.963823:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075386/real 1769075386] req@ffff9b0365a38000 x1855007617131392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075386.963837:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38000 x1855007617131392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075386.963847:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075386.963851:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075386.963854:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131392:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075386.963861:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131520:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075386.963884:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075386.963895:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075386/real 1769075386] req@ffff9b0365a38a80 x1855007617131520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075386.963915:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617131520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075386.963923:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075386.963927:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075386.963930:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131520:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075386.963937:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131648:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075386.963958:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075386.963969:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075386/real 1769075386] req@ffff9b0365a39c00 x1855007617131648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075386.963981:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617131648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075386.963988:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075386.963991:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075386.963994:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131648:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075386.964000:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131776:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075386.964018:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075386.964027:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075386/real 1769075386] req@ffff9b0365a38e00 x1855007617131776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075386.964038:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617131776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075386.964051:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075386.964054:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075386.964058:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617131776:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075387.475329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075387.859411:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075388.499329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075388.883331:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075389.523330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075389.907491:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075390.547330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075390.931430:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075391.571322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075391.955447:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075392.083349:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075392.083375:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075392.083380:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075392.083390:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2381 00000100:00080000:1.0:1769075392.083400:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075392.083407:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075392.083461:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617131904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075392.083495:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075392.083499:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075392.083501:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075392.083505:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2381 00000100:00080000:1.0:1769075392.083509:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075392.083514:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075392.083523:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075392.083553:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617131904:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075392.083555:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617132032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075392.083573:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075392.083576:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075392.083578:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075392.083582:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2381 00000100:00080000:1.0:1769075392.083587:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075392.083590:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075392.083594:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075392.083619:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617132160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075392.083626:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075392.083628:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075392.083630:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075392.083632:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2381 00000100:00080000:1.0:1769075392.083634:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075392.083637:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075392.083640:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075392.083648:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617132288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075392.083654:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075392.083660:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075392.083680:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075392.083683:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075392.083691:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2381 00000100:00080000:1.0:1769075392.083697:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075392.083718:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075392.083727:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075392.083752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075392/real 1769075392] req@ffff9b0343deea00 x1855007617131904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075392.083760:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617132416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075392.083773:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617131904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075392.083783:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075392.083787:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075392.083790:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617131904:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075392.083810:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132032:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075392.083827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075392.083835:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075392/real 1769075392] req@ffff9b0343deed80 x1855007617132032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075392.083844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617132032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075392.083854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075392.083856:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075392.083859:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132032:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075392.083864:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132160:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075392.083883:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075392.083912:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075392/real 1769075392] req@ffff9b0343deca80 x1855007617132160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075392.083929:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617132160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075392.083940:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075392.083944:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075392.083949:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132160:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075392.083973:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132288:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075392.084006:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075392.084025:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075392/real 1769075392] req@ffff9b0343dece00 x1855007617132288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075392.084048:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617132288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075392.084062:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075392.084067:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075392.084072:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132288:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075392.084080:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132416:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075392.084103:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075392.084115:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075392/real 1769075392] req@ffff9b0343defb80 x1855007617132416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075392.084144:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617132416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075392.084152:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075392.084154:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075392.084157:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132416:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075392.595351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075392.979387:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075393.619366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075394.003783:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075394.643362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075395.027475:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075395.667378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075396.051441:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075396.691304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075397.075366:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075397.203330:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075397.203339:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075397.203343:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075397.203351:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2386 00000100:00080000:2.0:1769075397.203359:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075397.203364:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075397.203402:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617132544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075397.203435:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075397.203438:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075397.203440:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075397.203443:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2386 00000100:00080000:2.0:1769075397.203445:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075397.203449:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075397.203458:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075397.203468:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617132544:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075397.203469:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617132672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075397.203496:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075397.203500:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075397.203502:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075397.203507:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2386 00000100:00080000:2.0:1769075397.203511:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075397.203516:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075397.203521:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075397.203535:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617132800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075397.203545:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075397.203551:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075397.203556:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075397.203558:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075397.203562:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2386 00000100:00080000:2.0:1769075397.203567:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075397.203570:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075397.203574:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075397.203588:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617132928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075397.203588:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075397/real 1769075397] req@ffff9b03482c5f80 x1855007617132544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075397.203640:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075397.203643:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075397.203645:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075397.203650:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617132544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075397.203652:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2386 00000100:00080000:2.0:1769075397.203660:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075397.203664:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075397.203668:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:3.0:1769075397.203672:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075397.203676:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617132544:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769075397.203678:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075397.203694:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617133056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075397.203698:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132672:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075397.203723:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075397.203735:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075397/real 1769075397] req@ffff9b03482c6a00 x1855007617132672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075397.203743:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617132672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075397.203749:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075397.203751:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075397.203753:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132672:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075397.203757:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132800:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075397.203766:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075397.203772:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075397/real 1769075397] req@ffff9b03482c4000 x1855007617132800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075397.203779:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617132800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075397.203784:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075397.203786:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075397.203787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132800:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075397.203791:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132928:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075397.203801:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075397.203807:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075397/real 1769075397] req@ffff9b03482c7100 x1855007617132928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075397.203814:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617132928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075397.203819:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075397.203820:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075397.203822:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617132928:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075397.203832:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133056:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075397.203841:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075397.203846:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075397/real 1769075397] req@ffff9b03482c4700 x1855007617133056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075397.203854:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617133056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075397.203859:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075397.203861:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075397.203862:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133056:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075397.715348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075398.099504:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075398.739331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075399.123374:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075399.763517:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075400.147385:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075400.787328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075401.171392:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075401.811366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075402.195388:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075402.323345:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075402.323354:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075402.323359:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075402.323369:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2391 00000100:00080000:0.0:1769075402.323392:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075402.323399:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075402.323438:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617133184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075402.323474:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075402.323477:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075402.323478:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075402.323481:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2391 00000100:00080000:0.0:1769075402.323484:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075402.323488:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075402.323495:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075402.323506:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617133312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075402.323515:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075402.323518:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075402.323521:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075402.323528:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2391 00000100:00100000:3.0:1769075402.323528:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617133184:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075402.323552:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075402.323559:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075402.323563:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075402.323578:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b800 x1855007617133440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075402.323587:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075402.323592:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075402.323594:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075402.323598:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2391 00000100:00080000:0.0:1769075402.323606:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075402.323610:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075402.323616:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075402.323631:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617133568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075402.323641:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075402.323655:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075402.323659:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075402.323662:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075402.323668:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2391 00000100:00080000:0.0:1769075402.323673:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075402.323679:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075402.323684:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075402.323694:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075402/real 1769075402] req@ffff9b0365a3b480 x1855007617133184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075402.323701:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617133696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075402.323744:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617133184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075402.323756:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075402.323761:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075402.323765:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617133184:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075402.323791:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133312:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075402.323810:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075402.323819:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075402/real 1769075402] req@ffff9b0365a3aa00 x1855007617133312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075402.323829:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617133312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075402.323836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075402.323839:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075402.323842:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133312:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075402.323846:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133440:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075402.323857:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075402.323864:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075402/real 1769075402] req@ffff9b0365a3b800 x1855007617133440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075402.323877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b800 x1855007617133440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075402.323883:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075402.323885:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075402.323888:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133440:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075402.323896:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133568:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075402.323934:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075402.323948:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075402/real 1769075402] req@ffff9b0365a39500 x1855007617133568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075402.323967:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617133568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075402.323995:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075402.323999:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075402.324004:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133568:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075402.324010:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133696:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075402.324028:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075402.324037:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075402/real 1769075402] req@ffff9b0365a3bb80 x1855007617133696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075402.324046:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617133696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075402.324075:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075402.324078:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075402.324081:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133696:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075402.835314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075403.219421:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075403.859376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075404.243475:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075404.883394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075405.267509:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075405.907375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075406.291439:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075406.931394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075407.315659:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075407.443379:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075407.443391:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075407.443396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075407.443409:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2396 00000100:00080000:1.0:1769075407.443420:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075407.443430:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075407.443484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617133824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075407.443514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075407.443529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075407.443532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075407.443540:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617133824:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075407.443543:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2396 00000100:00080000:1.0:1769075407.443551:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075407.443557:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075407.443567:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075407.443585:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617133952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075407.443599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769075407.443602:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075407.443608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075407.443612:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075407.443619:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2396 00000100:00080000:1.0:1769075407.443625:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075407.443629:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075407.443635:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075407.443642:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075407/real 1769075407] req@ffff9b0343dee680 x1855007617133824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075407.443651:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617134080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075407.443661:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617133824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075407.443672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075407.443675:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075407.443680:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075407.443682:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769075407.443685:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075407.443688:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617133824:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075407.443694:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2396 00000100:00080000:1.0:1769075407.443700:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075407.443704:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075407.443709:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133952:192.168.204.160@tcp:38:kworker.0 00000008:00100000:1.0:1769075407.443711:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075407.443729:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617134208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075407.443730:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075407.443751:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075407/real 1769075407] req@ffff9b0343ded500 x1855007617133952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075407.443752:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075407.443764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075407.443768:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075407.443772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617133952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075407.443777:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2396 00000100:00080000:3.0:1769075407.443786:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075407.443787:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075407.443791:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075407.443797:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617133952:192.168.204.160@tcp:38:kworker.0 00000100:00080000:1.0:1769075407.443798:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075407.443805:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134080:192.168.204.160@tcp:38:kworker.0 00000008:00100000:1.0:1769075407.443808:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075407.443823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075407.443828:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617134336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075407.443834:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075407/real 1769075407] req@ffff9b0343def480 x1855007617134080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075407.443856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617134080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075407.443862:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075407.443865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075407.443867:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134080:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075407.443877:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134208:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075407.443889:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075407.443897:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075407/real 1769075407] req@ffff9b0343dee680 x1855007617134208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075407.443904:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617134208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075407.443909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075407.443911:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075407.443914:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134208:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075407.443917:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134336:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075407.443926:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075407.443931:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075407/real 1769075407] req@ffff9b0343defb80 x1855007617134336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075407.443938:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617134336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075407.443943:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075407.443945:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075407.443946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134336:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075407.955355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075408.339398:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075408.979331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075409.363547:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075410.003325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075410.387377:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075411.027457:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075411.411443:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075412.051326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075412.435487:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075412.563444:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075412.563454:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075412.563458:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075412.563468:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2401 00000100:00080000:2.0:1769075412.563476:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075412.563483:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075412.563514:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617134464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075412.563545:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075412.563548:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075412.563550:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075412.563553:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2401 00000100:00080000:2.0:1769075412.563556:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075412.563559:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075412.563566:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075412.563575:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7800 x1855007617134592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075412.563585:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617134464:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075412.563586:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075412.563595:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075412.563596:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075412.563600:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2401 00000100:00080000:2.0:1769075412.563603:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075412.563606:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075412.563610:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075412.563618:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617134720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075412.563626:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075412.563629:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075412.563630:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075412.563632:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2401 00000100:00080000:2.0:1769075412.563636:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075412.563639:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075412.563643:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075412.563651:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075412.563671:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617134848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075412.563684:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075412.563688:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075412.563690:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075412.563694:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2401 00000100:00080000:2.0:1769075412.563697:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075412.563701:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075412.563706:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075412.563717:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075412/real 1769075412] req@ffff9b03482c5180 x1855007617134464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075412.563722:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617134976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075412.563765:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617134464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075412.563789:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075412.563794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075412.563798:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617134464:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075412.563820:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134592:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075412.563854:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075412.563864:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075412/real 1769075412] req@ffff9b03482c7800 x1855007617134592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075412.563876:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7800 x1855007617134592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075412.563883:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075412.563886:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075412.563889:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134592:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075412.563894:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134720:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075412.563907:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075412.563915:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075412/real 1769075412] req@ffff9b03482c6d80 x1855007617134720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075412.563925:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617134720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075412.563932:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075412.563934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075412.563937:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134720:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075412.563942:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134848:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075412.563966:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075412.563974:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075412/real 1769075412] req@ffff9b03482c6300 x1855007617134848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075412.563984:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617134848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075412.563990:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075412.563992:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075412.563995:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134848:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075412.564000:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134976:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075412.564012:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075412.564019:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075412/real 1769075412] req@ffff9b03482c4a80 x1855007617134976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075412.564028:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617134976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075412.564034:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075412.564037:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075412.564039:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617134976:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075413.075332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075413.459606:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075414.099355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075414.483422:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075415.123395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075415.507452:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075416.147366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075416.531545:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075417.171400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075417.555389:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075417.683373:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075417.683381:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075417.683386:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075417.683395:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2406 00000100:00080000:0.0:1769075417.683402:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075417.683409:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075417.683436:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617135104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075417.683457:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075417.683461:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075417.683462:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075417.683466:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2406 00000100:00080000:0.0:1769075417.683469:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075417.683472:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075417.683480:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075417.683491:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617135232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075417.683499:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617135104:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075417.683504:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075417.683508:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075417.683510:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075417.683514:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2406 00000100:00080000:0.0:1769075417.683518:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075417.683521:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075417.683524:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075417.683535:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617135360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075417.683546:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075417.683548:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075417.683550:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075417.683552:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2406 00000100:00080000:0.0:1769075417.683556:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075417.683558:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075417.683562:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075417.683568:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075417.683570:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617135488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075417.683578:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075417.683581:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075417.683583:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075417.683585:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2406 00000100:00080000:0.0:1769075417.683588:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075417.683591:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075417.683594:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075417.683602:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617135616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075417.683610:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075417/real 1769075417] req@ffff9b0365a39180 x1855007617135104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075417.683626:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617135104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075417.683635:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075417.683638:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075417.683642:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617135104:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075417.683660:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617135232:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075417.683687:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075417.683702:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075417/real 1769075417] req@ffff9b0365a3ad80 x1855007617135232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075417.683716:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617135232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075417.683725:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075417.683727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075417.683730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617135232:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075417.683735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617135360:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075417.683750:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075417.683758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075417/real 1769075417] req@ffff9b0365a3b100 x1855007617135360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075417.683766:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617135360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075417.683771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075417.683773:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075417.683775:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617135360:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075417.683779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617135488:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075417.683789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075417.683794:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075417/real 1769075417] req@ffff9b0365a3bb80 x1855007617135488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075417.683806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617135488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075417.683811:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075417.683813:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075417.683815:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617135488:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075417.683818:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617135616:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075417.683827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075417.683832:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075417/real 1769075417] req@ffff9b0365a39500 x1855007617135616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075417.683838:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617135616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075417.683844:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075417.683846:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075417.683848:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617135616:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075418.195443:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075418.579369:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075419.219340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075419.603428:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075420.243335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075420.627440:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075421.267433:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075421.651370:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075422.291376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075422.675403:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075422.803365:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075422.803374:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075422.803378:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075422.803387:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2411 00000100:00080000:2.0:1769075422.803395:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075422.803400:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075422.803434:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617135744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075422.803460:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075422.803463:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075422.803465:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075422.803468:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2411 00000100:00080000:2.0:1769075422.803471:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075422.803473:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075422.803480:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075422.803490:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617135872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075422.803498:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075422.803501:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075422.803502:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075422.803505:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2411 00000100:00080000:2.0:1769075422.803507:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075422.803514:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617135744:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075422.803514:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075422.803518:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075422.803532:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617136000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075422.803539:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075422.803542:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075422.803543:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075422.803546:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2411 00000100:00080000:2.0:1769075422.803548:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075422.803550:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075422.803553:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075422.803561:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617136128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075422.803576:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075422.803578:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075422.803580:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075422.803583:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2411 00000100:00080000:2.0:1769075422.803586:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075422.803588:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075422.803592:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075422.803596:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075422.803604:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617136256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075422.803643:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075422/real 1769075422] req@ffff9b03482c5500 x1855007617135744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075422.803660:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617135744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075422.803669:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075422.803672:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075422.803675:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617135744:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075422.803681:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617135872:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075422.803701:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075422.803710:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075422/real 1769075422] req@ffff9b03482c4380 x1855007617135872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075422.803729:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617135872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075422.803737:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075422.803741:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075422.803745:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617135872:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075422.803770:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136000:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075422.803793:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075422.803805:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075422/real 1769075422] req@ffff9b03482c4a80 x1855007617136000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075422.803816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617136000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075422.803826:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075422.803829:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075422.803833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136000:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075422.803840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136128:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075422.803862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075422.803874:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075422/real 1769075422] req@ffff9b03482c6300 x1855007617136128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075422.803884:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617136128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075422.803890:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075422.803893:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075422.803895:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136128:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075422.803900:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136256:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075422.803913:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075422.803919:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075422/real 1769075422] req@ffff9b03482c6d80 x1855007617136256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075422.803927:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617136256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075422.803932:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075422.803934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075422.803936:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136256:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075423.315381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075423.699399:0:28211:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 3/3 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1769075423.699456:0:57730:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 3 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769075423.699465:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x28400020000 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769075423.699470:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075423.699503:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400020000 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000080:00200000:0.0:1769075423.699522:0:57730:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x204:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:0.0:1769075423.699546:0:57730:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x204:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:0.0:1769075423.699558:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x28480020000 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769075423.699607:0:57730:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400020020 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769075423.699621:0:57730:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400020020 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769075423.699632:0:57730:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 1/0,0 mode: --/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4829480020020 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769075423.699640:0:57730:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1769075423.699671:0:57730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292400/0xb7d6bf619879c633 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x4829400020020 nid: local remote: 0x7fe2323adfd54fe0 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769075423.699699:0:57730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0349292000/0xb7d6bf619879c625 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400020020 nid: local remote: 0x7fe2323adfd54fcb expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000100:00080000:1.0:1769075423.699711:0:2407:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9b0365a3b800 x1855007617136384/t0(0) o103->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:17/18 lens 336/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:0.0:1769075423.699714:0:57730:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff9b0342ad2800 lock: ffff9b0342cd1a00/0xb7d6bf619879c53e lrc: 0/0,0 mode: --/PW res: [0x280000401:0x2c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4829480020020 nid: local remote: 0x7fe2323adfd54cc9 expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:0.0:1769075423.699835:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075423.699839:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075423.699842:0:57730:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075424.339365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075424.723509:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075425.363408:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075425.747474:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075426.387418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075426.771427:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075427.411406:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075427.795498:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075427.923462:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075427.923473:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075427.923478:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075427.923488:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2417 00000100:00080000:0.0:1769075427.923498:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075427.923506:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075427.923546:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617136512/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075427.923577:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075427.923581:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075427.923583:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075427.923587:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2417 00000100:00080000:0.0:1769075427.923590:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075427.923594:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075427.923602:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075427.923614:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617136640/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075427.923626:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075427.923631:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1769075427.923635:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617136512:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075427.923636:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075427.923648:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2417 00000100:00080000:0.0:1769075427.923652:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075427.923656:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075427.923662:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075427.923676:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617136768/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075427.923685:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075427.923689:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075427.923690:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075427.923694:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2417 00000100:00080000:0.0:1769075427.923706:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075427.923713:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769075427.923715:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769075427.923723:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075427.923738:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617136896/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075427.923748:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075427.923751:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075427.923753:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075427.923757:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2417 00000100:00080000:0.0:1769075427.923762:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075427.923768:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075427/real 1769075427] req@ffff9b0365a3aa00 x1855007617136512/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075427.923769:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075427.923778:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075427.923796:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617137024/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075427.923804:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617136512/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075427.923825:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075427.923830:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075427.923836:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617136512:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075427.923860:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136640:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075427.923881:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075427.923907:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075427/real 1769075427] req@ffff9b0365a3b480 x1855007617136640/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075427.923919:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617136640/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075427.923927:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075427.923930:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075427.923933:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136640:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075427.923940:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075427.923959:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075427.923967:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075427/real 1769075427] req@ffff9b0365a38e00 x1855007617136768/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075427.923978:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617136768/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075427.923985:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075427.923987:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075427.923990:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075427.923995:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136896:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075427.924009:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075427.924017:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075427/real 1769075427] req@ffff9b0365a39c00 x1855007617136896/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075427.924027:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617136896/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075427.924035:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075427.924037:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075427.924040:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617136896:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075427.924045:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617137024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075427.924057:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075427.924065:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075427/real 1769075427] req@ffff9b0365a38a80 x1855007617137024/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075427.924082:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617137024/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075427.924089:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075427.924092:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075427.924094:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617137024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075428.435323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075428.819545:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075429.459313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075429.843531:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075430.483409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075430.867801:0:28211:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 16/16 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1769075430.867885:0:57730:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 16 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:0.0:1769075430.867931:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005aa00/0xb7d6bf619879c982 lrc: 2/0,0 mode: PR/PR res: [0x200000bd0:0xa:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55d38 expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075430.868008:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005ae00/0xb7d6bf619879c997 lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55d69 expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075430.868042:0:57730:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075430.868134:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005a200/0xb7d6bf619879c97b lrc: 2/0,0 mode: PR/PR res: [0x200000405:0x20c:0x0].0x0 bits 0x11/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55d2a expref: -99 pid: 68436 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075430.868158:0:57730:0:(ldlm_lock.c:1515:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00010000:00010000:2.0:1769075430.868162:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075430.868186:0:57730:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035005b800/0xb7d6bf619879c8e1 lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x209:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55a75 expref: -99 pid: 67448 timeout: 0 lvb_type: 0 00000002:00100000:0.0:1769075430.868376:0:57730:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0352a19880 x1855007617041920/t8589944129(8589944129) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074936 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:0.0:1769075430.868402:0:57730:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 2 is allocated opc 35, max 7 00000100:00100000:0.0:1769075430.868413:0:57730:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b0365a38000 x1855007617137280/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00080000:0.0:1769075430.868453:0:57730:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9b0365a38000 x1855007617137280/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1769075430.868476:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075431.507349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769075431.891454:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075431.891645:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075432.531350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769075432.915472:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075432.915503:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:1.0:1769075433.043410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075433.043419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075433.043423:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075433.043431:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2422 00000100:00080000:1.0:1769075433.043440:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075433.043445:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075433.043476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617137408/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075433.043500:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075433.043503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075433.043504:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075433.043507:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2422 00000100:00080000:1.0:1769075433.043509:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075433.043512:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075433.043519:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075433.043527:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617137536/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075433.043535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075433.043537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075433.043538:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075433.043541:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2422 00000100:00080000:1.0:1769075433.043542:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075433.043544:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075433.043547:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075433.043563:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617137664/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075433.043569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075433.043571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075433.043572:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075433.043575:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2422 00000100:00080000:1.0:1769075433.043583:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075433.043585:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075433.043589:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075433.043596:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617137792/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075433.043603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075433.043606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1769075433.043606:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617137408:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075433.043610:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075433.043618:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2422 00000100:00080000:1.0:1769075433.043620:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075433.043623:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075433.043626:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075433.043635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617137920/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075433.043671:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075433.043704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff9b0343dece00 x1855007617137408/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.043718:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617137408/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.043725:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.043728:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075433.043731:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617137408:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075433.043736:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617137536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075433.043765:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075433.043775:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff9b0343deca80 x1855007617137536/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.043784:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617137536/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.043791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.043794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075433.043796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617137536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075433.043800:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617137664:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075433.043812:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075433.043818:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff9b0343deed80 x1855007617137664/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.043827:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617137664/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.043833:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.043835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075433.043837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617137664:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075433.043859:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617137792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075433.043871:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075433.043878:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff9b0343deea00 x1855007617137792/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.043893:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617137792/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.043898:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.043900:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075433.043903:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617137792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075433.043907:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617137920:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075433.043918:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075433.043924:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075433/real 1769075433] req@ffff9b0343def800 x1855007617137920/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075433.043933:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617137920/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075433.043938:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075433.043940:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075433.043942:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617137920:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075433.555356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075433.939413:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075433.939466:0:28211:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 2/2 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1769075433.939598:0:39340:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 2 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1769075433.939612:0:39340:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x800028400000000 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:3.0:1769075433.939612:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1769075433.939650:0:39340:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x200:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769075433.940962:0:39340:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x200:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769075433.940983:0:39340:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800028400020000 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000080:00200000:1.0:1769075433.940995:0:39340:0:(vvp_io.c:1854:vvp_io_init()) [0x200000405:0x204:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1769075433.941000:0:39340:0:(vvp_io.c:289:vvp_io_fini()) [0x200000405:0x204:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:1.0:1769075433.941050:0:39340:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804869400000020 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769075433.941057:0:39340:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400020020 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769075433.941079:0:39340:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1769075433.941106:0:39340:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0342cd1000/0xb7d6bf619879c54c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x2c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 1048576->2097151) gid 0 flags: 0x804869400000020 nid: local remote: 0x7fe2323adfd54cec expref: -99 pid: 64903 timeout: 0 lvb_type: 1 00010000:00010000:1.0:1769075433.941149:0:39340:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffff9b0342ad2800 lock: ffff9b0349293a00/0xb7d6bf619879c62c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x2c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x804869400020020 nid: local remote: 0x7fe2323adfd54fd2 expref: -99 pid: 65495 timeout: 0 lvb_type: 1 00000100:00080000:0.0:1769075433.941163:0:2406:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9b0343dedc00 x1855007617138048/t0(0) o103->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1769075433.941164:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075433.941169:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1769075433.941173:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075434.579310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075434.963336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075434.963473:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075435.603322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075435.987351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075435.987411:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075436.627362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075437.011329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075437.011476:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075437.651321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075438.035405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075438.035461:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075438.163436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075438.163452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075438.163459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075438.163490:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2427 00000100:00080000:2.0:1769075438.163506:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075438.163516:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075438.163599:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7800 x1855007617138176/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075438.163643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075438.163646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075438.163648:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075438.163652:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2427 00000100:00080000:2.0:1769075438.163655:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075438.163659:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075438.163669:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075438.163681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617138304/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075438.163707:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075438.163711:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075438.163713:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075438.163718:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2427 00000100:00080000:2.0:1769075438.163721:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075438.163725:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075438.163731:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075438.163736:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617138176:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075438.163746:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617138432/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075438.163771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075438.163774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075438.163776:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075438.163780:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2427 00000100:00080000:2.0:1769075438.163785:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075438.163789:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075438.163796:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075438.163808:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617138560/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075438.163818:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075438.163822:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075438.163824:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075438.163827:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2427 00000100:00080000:2.0:1769075438.163849:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075438.163853:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075438.163858:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075438.163870:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617138688/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075438.163882:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075438.163972:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075438/real 1769075438] req@ffff9b03482c7800 x1855007617138176/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075438.164005:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7800 x1855007617138176/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075438.164023:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075438.164030:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075438.164037:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617138176:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075438.164067:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617138304:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075438.164108:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075438.164131:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075438/real 1769075438] req@ffff9b03482c5180 x1855007617138304/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075438.164152:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617138304/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075438.164168:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075438.164174:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075438.164180:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617138304:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075438.164236:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617138432:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075438.164305:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075438.164338:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075438/real 1769075438] req@ffff9b03482c4700 x1855007617138432/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075438.164358:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617138432/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075438.164367:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075438.164370:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075438.164374:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617138432:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075438.164380:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617138560:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075438.164425:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075438.164443:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075438/real 1769075438] req@ffff9b03482c7100 x1855007617138560/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075438.164479:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617138560/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075438.164491:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075438.164496:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075438.164501:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617138560:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075438.164510:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617138688:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075438.164538:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075438.164553:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075438/real 1769075438] req@ffff9b03482c4000 x1855007617138688/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075438.164584:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617138688/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075438.164599:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075438.164603:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075438.164606:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617138688:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075438.675428:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075439.059397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075439.059502:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075439.699422:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075440.083403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075440.083499:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075440.723355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075441.107308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075441.107385:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075441.747354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075442.131337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075442.131421:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075442.771320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075443.155418:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075443.155433:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075443.283363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075443.283374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075443.283380:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075443.283390:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2432 00000100:00080000:2.0:1769075443.283402:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075443.283409:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075443.283466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617138816/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075443.283498:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075443.283502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075443.283504:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075443.283509:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2432 00000100:00080000:2.0:1769075443.283512:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075443.283516:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075443.283525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075443.283545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617138944/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075443.283561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075443.283567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075443.283570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075443.283575:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617138816:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075443.283582:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2432 00000100:00080000:2.0:1769075443.283592:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075443.283599:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075443.283607:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075443.283625:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617139072/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075443.283638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075443.283642:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075443.283644:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075443.283648:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2432 00000100:00080000:2.0:1769075443.283653:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769075443.283656:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075443.283662:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075443.283669:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075443.283681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617139200/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075443.283692:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075443.283696:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075443.283697:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075443.283702:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2432 00000100:00080000:2.0:1769075443.283705:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075443.283709:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075443.283713:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075443.283721:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075443/real 1769075443] req@ffff9b03482c6a00 x1855007617138816/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075443.283728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617139328/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075443.283770:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617138816/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075443.283784:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075443.283789:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075443.283793:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617138816:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075443.283801:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617138944:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075443.283823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075443.283835:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075443/real 1769075443] req@ffff9b03482c5f80 x1855007617138944/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075443.283892:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617138944/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075443.283900:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075443.283903:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075443.283906:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617138944:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075443.283931:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139072:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075443.283968:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075443.283979:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075443/real 1769075443] req@ffff9b03482c4e00 x1855007617139072/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075443.283991:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617139072/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075443.283999:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075443.284002:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075443.284005:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139072:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075443.284010:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139200:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075443.284024:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075443.284048:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075443/real 1769075443] req@ffff9b03482c7b80 x1855007617139200/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075443.284059:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617139200/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075443.284072:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075443.284075:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075443.284077:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139200:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075443.284082:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139328:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075443.284096:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075443.284103:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075443/real 1769075443] req@ffff9b03482c5c00 x1855007617139328/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075443.284113:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617139328/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075443.284120:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075443.284122:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075443.284125:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139328:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075443.795359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075444.179384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075444.179462:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075444.819365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075445.203319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075445.203405:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075445.843325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075446.227374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075446.227452:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075446.867366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075447.251303:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075447.251429:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075447.891314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075448.275335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075448.275406:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075448.403310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075448.403322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075448.403326:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075448.403337:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2437 00000100:00080000:2.0:1769075448.403347:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075448.403355:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075448.403397:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617139456/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075448.403425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075448.403427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075448.403429:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075448.403432:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2437 00000100:00080000:2.0:1769075448.403434:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075448.403437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075448.403445:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075448.403455:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617139584/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075448.403476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075448.403480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075448.403481:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075448.403484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2437 00000100:00080000:2.0:1769075448.403486:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075448.403491:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617139456:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075448.403492:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075448.403499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075448.403511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617139712/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075448.403522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075448.403525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075448.403536:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075448.403539:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2437 00000100:00080000:2.0:1769075448.403551:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075448.403554:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075448.403558:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075448.403566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617139840/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075448.403581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075448.403585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075448.403587:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075448.403593:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2437 00000100:00100000:3.0:1769075448.403594:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139584:192.168.204.160@tcp:38:kworker.0 00000100:00080000:2.0:1769075448.403599:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075448.403607:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075448.403612:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075448.403626:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617139968/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075448.403629:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139712:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075448.403654:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139840:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075448.403670:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139968:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075448.915378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075449.299324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075449.299372:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075449.939336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075450.323340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075450.323366:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075450.963322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075451.347405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075451.347419:0:28211:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 3/3 locks to regular blp list, 1 blwis in pool 00010000:00010000:0.0:1769075451.347489:0:39340:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 3 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1769075451.347503:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075451.347509:0:39340:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 2/0,0 mode: PR/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075451.347549:0:39340:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 2/0,0 mode: PR/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075451.347561:0:39340:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860c00/0xb7d6bf619879c76e lrc: 2/0,0 mode: PR/PR res: [0x240000405:0x8:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd554d3 expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075451.347616:0:39340:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 1/0,0 mode: --/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075451.347624:0:39340:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 1/0,0 mode: --/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075451.347630:0:39340:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860c00/0xb7d6bf619879c76e lrc: 1/0,0 mode: --/PR res: [0x240000405:0x8:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd554d3 expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075451.347648:0:39340:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:0.0:1769075451.347677:0:39340:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035919ea00/0xb7d6bf619879cab6 lrc: 0/0,0 mode: --/PR res: [0x240000405:0xb:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd561a6 expref: -99 pid: 69562 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075451.347697:0:39340:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b035005b200/0xb7d6bf619879c894 lrc: 0/0,0 mode: --/PR res: [0x240000405:0x9:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd5596b expref: -99 pid: 67398 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075451.347736:0:39340:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff9b0342ad2800 lock: ffff9b0365860c00/0xb7d6bf619879c76e lrc: 0/0,0 mode: --/PR res: [0x240000405:0x8:0x0].0x0 bits 0x12/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x7fe2323adfd554d3 expref: -99 pid: 66726 timeout: 0 lvb_type: 0 00010000:00010000:0.0:1769075451.347760:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075451.347764:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1769075451.347768:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075451.347779:0:2407:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff9b0365a39f80 x1855007617140096/t0(0) o103->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:17/18 lens 336/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000400:00000100:3.0:1769075451.475839:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:3.0:1769075451.475909:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769075451.475938:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075451.476022:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075451.476038:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075451.476055:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075451.476073:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769075451.476131:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075448/real 1769075451] req@ffff9b03482c4e00 x1855007617139968/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075451.476201:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617139968/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075451.476217:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075451.476225:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075451.476277:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139968:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075451.476289:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075448/real 1769075451] req@ffff9b03482c5c00 x1855007617139712/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075451.476315:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617139712/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075451.476327:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075451.476335:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075451.476343:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139712:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075451.476353:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075448/real 1769075451] req@ffff9b03482c7b80 x1855007617139840/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075451.476374:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617139840/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075451.476387:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075451.476396:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075451.476402:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139840:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075451.476414:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075448/real 1769075451] req@ffff9b03482c5880 x1855007617139456/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075451.476430:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617139456/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075451.476438:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075451.476442:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075451.476445:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617139456:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075451.476455:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075448/real 1769075451] req@ffff9b03482c6680 x1855007617139584/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075451.476474:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617139584/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075451.476484:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075451.476489:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075451.476495:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617139584:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075451.987385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075452.371441:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075452.371471:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075453.011319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075453.395375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075453.395489:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075453.523372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075453.523383:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075453.523387:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075453.523397:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2442 00000100:00080000:1.0:1769075453.523412:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075453.523420:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075453.523451:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617140224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075453.523506:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075453.523510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075453.523512:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075453.523518:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2442 00000100:00080000:1.0:1769075453.523522:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075453.523527:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075453.523538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075453.523552:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617140224:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075453.523562:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617140352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075453.523583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075453.523589:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075453.523593:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075453.523602:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2442 00000100:00080000:1.0:1769075453.523611:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075453.523634:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075453.523642:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075453.523657:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075453.523662:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617140480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075453.523698:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075453.523702:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075453.523705:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075453.523711:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2442 00000100:00080000:1.0:1769075453.523718:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075453.523720:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075453/real 1769075453] req@ffff9b0343ded880 x1855007617140224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075453.523727:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075453.523736:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075453.523755:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617140608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075453.523760:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617140224/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075453.523769:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075453.523776:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075453.523778:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075453.523779:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075453.523786:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2442 00000100:00080000:2.0:1769075453.523788:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769075453.523795:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075453.523797:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617140224:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075453.523807:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075453.523816:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075453.523833:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617140736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075453.523838:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617140352:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075453.523889:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075453.523908:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075453/real 1769075453] req@ffff9b0343def100 x1855007617140352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075453.523924:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617140352/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075453.523934:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075453.523939:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075453.523943:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617140352:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075453.523951:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617140480:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075453.523975:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075453.523989:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075453/real 1769075453] req@ffff9b0343dee300 x1855007617140480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075453.524004:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617140480/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075453.524015:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075453.524019:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075453.524023:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617140480:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075453.524030:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617140608:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075453.524057:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075453.524070:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075453/real 1769075453] req@ffff9b0343dec380 x1855007617140608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075453.524088:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617140608/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075453.524098:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075453.524102:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075453.524106:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617140608:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075453.524134:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617140736:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075453.524161:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075453.524175:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075453/real 1769075453] req@ffff9b0343ded180 x1855007617140736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075453.524190:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617140736/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075453.524201:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075453.524205:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075453.524209:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617140736:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075454.035327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075454.419398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075454.419482:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075455.059307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075455.443454:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075455.443565:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075456.083424:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075456.467362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075456.467482:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075457.107351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075457.491347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075457.491418:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075458.131356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075458.515391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075458.515641:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075458.643395:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075458.643409:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075458.643415:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075458.643428:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2447 00000100:00080000:0.0:1769075458.643441:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075458.643450:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075458.643497:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617140864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075458.643555:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075458.643563:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075458.643566:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075458.643575:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2447 00000100:00080000:0.0:1769075458.643583:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075458.643612:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075458.643629:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075458.643629:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617140864:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075458.643673:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617140992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075458.643691:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075458.643696:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075458.643699:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075458.643704:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2447 00000100:00080000:0.0:1769075458.643710:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075458.643714:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075458.643721:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075458.643741:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617141120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075458.643776:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769075458.643785:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075458.643787:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075458.643795:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075458.643804:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2447 00000100:00080000:0.0:1769075458.643813:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075458.643821:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075458.643831:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075458.643861:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617141248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075458.643862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075458/real 1769075458] req@ffff9b0365a38a80 x1855007617140864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075458.643906:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075458.643921:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075458.643928:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075458.643938:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2447 00000100:00080000:0.0:1769075458.643946:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075458.643952:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617140864/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075458.643959:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075458.643974:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769075458.643984:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769075458.643997:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617141376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075458.643998:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075458.644032:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617140864:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075458.644087:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617140992:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075458.644161:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075458.644198:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075458/real 1769075458] req@ffff9b0365a39c00 x1855007617140992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075458.644228:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617140992/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075458.644293:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075458.644300:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075458.644307:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617140992:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075458.644320:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141120:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075458.644360:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075458.644383:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075458/real 1769075458] req@ffff9b0365a38e00 x1855007617141120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075458.644409:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617141120/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075458.644431:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075458.644437:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075458.644444:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141120:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075458.644456:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141248:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075458.644509:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075458.644528:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075458/real 1769075458] req@ffff9b0365a3b480 x1855007617141248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075458.644549:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617141248/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075458.644563:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075458.644568:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075458.644574:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141248:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075458.644585:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141376:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075458.644662:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075458.644686:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075458/real 1769075458] req@ffff9b0365a3aa00 x1855007617141376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075458.644703:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617141376/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075458.644714:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075458.644718:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075458.644723:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141376:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075459.155365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075459.539380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075459.539520:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075460.179354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075460.563401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075460.563490:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075461.203387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075461.587505:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075461.587628:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075462.227508:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075462.611383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075462.611516:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075463.251412:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075463.635352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075463.635497:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075463.763361:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075463.763383:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075463.763391:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075463.763410:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2452 00000100:00080000:0.0:1769075463.763429:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075463.763442:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075463.763558:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617141504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075463.763602:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075463.763619:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075463.763621:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075463.763627:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2452 00000100:00080000:0.0:1769075463.763633:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075463.763638:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075463.763653:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075463.763682:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617141632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075463.763700:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075463.763705:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075463.763710:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769075463.763717:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617141504:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075463.763719:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2452 00000100:00080000:0.0:1769075463.763737:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075463.763746:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075463.763751:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075463.763769:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617141760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075463.763782:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075463.763789:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075463.763791:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075463.763795:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2452 00000100:00080000:0.0:1769075463.763800:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075463.763804:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075463.763824:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075463.763828:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075463.763848:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617141888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075463.763865:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075463.763870:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075463.763874:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075463.763883:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2452 00000100:00080000:0.0:1769075463.763891:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769075463.763893:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075463/real 1769075463] req@ffff9b0365a39500 x1855007617141504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075463.763901:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075463.763912:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075463.763930:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617142016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075463.763949:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617141504/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075463.763973:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075463.763979:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075463.763985:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617141504:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075463.764015:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141632:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075463.764045:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075463.764059:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075463/real 1769075463] req@ffff9b0365a3bb80 x1855007617141632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075463.764071:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617141632/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075463.764080:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075463.764084:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075463.764087:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141632:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075463.764094:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141760:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075463.764111:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075463.764120:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075463/real 1769075463] req@ffff9b0365a3b100 x1855007617141760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075463.764131:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617141760/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075463.764138:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075463.764141:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075463.764144:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141760:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075463.764149:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141888:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075463.764163:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075463.764171:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075463/real 1769075463] req@ffff9b0365a3ad80 x1855007617141888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075463.764181:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617141888/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075463.764189:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075463.764192:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075463.764194:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617141888:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075463.764200:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142016:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075463.764238:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075463.764287:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075463/real 1769075463] req@ffff9b0365a39180 x1855007617142016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075463.764312:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617142016/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075463.764323:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075463.764327:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075463.764331:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142016:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075464.275389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075464.659382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075464.659556:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075465.299327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075465.683346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075465.683528:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075466.323398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075466.707395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075466.707436:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075467.347358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075467.731373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075467.731492:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075468.371317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075468.755394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075468.755505:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075468.883396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075468.883409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075468.883433:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075468.883448:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2458 00000100:00080000:1.0:1769075468.883459:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075468.883468:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075468.883510:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617142144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075468.883546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075468.883551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075468.883553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075468.883558:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2458 00000100:00080000:1.0:1769075468.883562:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075468.883565:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075468.883576:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075468.883589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617142272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075468.883600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075468.883605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075468.883607:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075468.883612:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2458 00000100:00080000:1.0:1769075468.883617:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075468.883622:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075468.883627:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075468.883639:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617142144:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075468.883642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617142400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075468.883674:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075468.883681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075468.883685:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075468.883694:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2458 00000100:00080000:1.0:1769075468.883701:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075468.883708:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075468.883719:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075468.883744:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617142528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075468.883752:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075468.883772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075468.883776:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075468.883779:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075468.883786:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2458 00000100:00080000:1.0:1769075468.883793:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075468.883799:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075468.883807:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075468.883823:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075468/real 1769075468] req@ffff9b0343dec000 x1855007617142144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075468.883825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617142656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075468.883868:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617142144/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075468.883918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075468.883925:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075468.883931:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617142144:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075468.883941:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142272:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075468.883963:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075468.883973:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075468/real 1769075468] req@ffff9b0343dec700 x1855007617142272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075468.883982:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617142272/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075468.883989:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075468.883991:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075468.883994:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142272:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075468.884021:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142400:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075468.884032:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075468.884039:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075468/real 1769075468] req@ffff9b0343dedf80 x1855007617142400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075468.884046:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617142400/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075468.884052:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075468.884054:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075468.884056:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142400:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075468.884061:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142528:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075468.884072:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075468.884078:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075468/real 1769075468] req@ffff9b0343ded180 x1855007617142528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075468.884086:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617142528/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075468.884091:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075468.884093:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075468.884095:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142528:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075468.884100:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142656:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075468.884110:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075468.884116:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075468/real 1769075468] req@ffff9b0343dec380 x1855007617142656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075468.884122:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617142656/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075468.884128:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075468.884130:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075468.884132:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142656:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075469.395370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075469.779354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075469.779472:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075470.419441:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075470.803348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075470.803488:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075471.443478:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075471.827420:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075471.827536:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075472.467386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769075472.851440:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075472.851443:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075473.491437:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075473.875414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075473.875473:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075474.003357:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075474.003371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075474.003377:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075474.003388:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2463 00000100:00080000:2.0:1769075474.003400:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075474.003408:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075474.003452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617142784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075474.003486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075474.003490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075474.003493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075474.003499:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2463 00000100:00080000:2.0:1769075474.003504:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075474.003510:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075474.003523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075474.003540:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617142784:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075474.003542:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617142912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075474.003567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075474.003570:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075474.003571:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075474.003576:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2463 00000100:00080000:2.0:1769075474.003579:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075474.003584:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075474.003590:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075474.003606:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617143040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075474.003615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075474.003618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075474.003620:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075474.003623:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2463 00000100:00080000:2.0:1769075474.003625:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075474.003633:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769075474.003641:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769075474.003644:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075474.003659:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617143168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075474.003668:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075474.003671:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075474.003672:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075474.003680:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2463 00000100:00080000:2.0:1769075474.003684:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075474.003685:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9b03482c5f80 x1855007617142784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075474.003697:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075474.003710:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075474.003725:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617142784/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075474.003731:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617143296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075474.003791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075474.003800:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075474.003806:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617142784:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075474.003833:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142912:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075474.003855:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075474.003865:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9b03482c6a00 x1855007617142912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075474.003882:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617142912/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075474.003889:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075474.003891:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075474.003894:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617142912:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075474.003899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143040:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075474.003912:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075474.003919:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9b03482c4000 x1855007617143040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075474.003933:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617143040/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075474.003938:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075474.003940:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075474.003942:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143040:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075474.003947:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143168:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075474.003958:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075474.003964:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9b03482c7100 x1855007617143168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075474.003971:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617143168/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075474.003976:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075474.003978:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075474.003980:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143168:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075474.003983:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143296:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075474.003994:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075474.004000:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075474/real 1769075474] req@ffff9b03482c4700 x1855007617143296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075474.004007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617143296/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075474.004012:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075474.004014:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075474.004016:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143296:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075474.515451:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075474.899405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075474.899512:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075475.539440:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075475.923414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075475.923548:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075476.563453:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075476.947365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075476.947515:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075477.587411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075477.971346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075477.971398:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075478.611405:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075478.995398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075478.995545:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075479.123369:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075479.123382:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075479.123388:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075479.123401:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2468 00000100:00080000:2.0:1769075479.123416:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075479.123425:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075479.123502:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617143424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075479.123547:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075479.123551:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075479.123554:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075479.123559:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2468 00000100:00080000:2.0:1769075479.123563:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075479.123567:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075479.123579:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075479.123608:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7800 x1855007617143552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075479.123617:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617143424:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075479.123620:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075479.123627:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075479.123629:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075479.123633:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2468 00000100:00080000:2.0:1769075479.123636:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075479.123639:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075479.123643:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075479.123653:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617143680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075479.123659:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075479.123662:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075479.123664:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075479.123672:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2468 00000100:00080000:2.0:1769075479.123675:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075479.123677:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075479.123681:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075479.123690:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617143808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075479.123696:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075479.123699:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075479.123701:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075479.123707:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2468 00000800:00000100:1.0:1769075479.123708:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075479.123711:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075479.123714:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075479.123717:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075479.123726:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617143936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075479.123763:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075479/real 1769075479] req@ffff9b03482c5180 x1855007617143424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075479.123781:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617143424/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075479.123791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075479.123795:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075479.123799:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617143424:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075479.123834:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143552:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075479.123870:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075479.123881:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075479/real 1769075479] req@ffff9b03482c7800 x1855007617143552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075479.123891:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7800 x1855007617143552/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075479.123898:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075479.123901:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075479.123904:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143552:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075479.123908:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143680:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075479.123931:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075479.123938:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075479/real 1769075479] req@ffff9b03482c6d80 x1855007617143680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075479.123947:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617143680/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075479.123970:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075479.123972:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075479.123974:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143680:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075479.123978:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143808:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075479.123991:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075479.123997:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075479/real 1769075479] req@ffff9b03482c6300 x1855007617143808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075479.124005:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617143808/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075479.124011:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075479.124013:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075479.124015:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143808:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075479.124018:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143936:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075479.124029:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075479.124034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075479/real 1769075479] req@ffff9b03482c4a80 x1855007617143936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075479.124042:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617143936/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075479.124048:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075479.124049:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075479.124051:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617143936:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075479.635322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075480.019338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075480.019395:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075480.659360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075481.043418:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075481.043559:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075481.683449:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075482.067304:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075482.067411:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075482.707333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075483.091357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075483.091405:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075483.731310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075484.115301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075484.115367:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075484.243361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075484.243376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075484.243382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075484.243396:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2473 00000100:00080000:2.0:1769075484.243409:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075484.243418:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075484.243490:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617144064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075484.243529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075484.243534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075484.243536:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075484.243543:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2473 00000100:00080000:2.0:1769075484.243548:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075484.243553:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075484.243566:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075484.243581:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617144192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075484.243594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075484.243598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075484.243602:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075484.243608:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2473 00000100:00080000:2.0:1769075484.243613:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075484.243618:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075484.243625:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075484.243634:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617144064:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075484.243648:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617144320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075484.243666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075484.243670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075484.243673:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075484.243679:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2473 00000100:00080000:2.0:1769075484.243685:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075484.243691:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075484.243699:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075484.243709:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075484.243718:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617144448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075484.243730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075484.243732:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075484.243734:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075484.243739:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2473 00000100:00080000:2.0:1769075484.243745:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075484.243749:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075484.243753:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075484/real 1769075484] req@ffff9b03482c4380 x1855007617144064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769075484.243767:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075484.243779:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617144064/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075484.243784:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617144576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075484.243844:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075484.243851:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075484.243856:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617144064:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075484.243864:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144192:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075484.243887:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075484.243899:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075484/real 1769075484] req@ffff9b03482c5500 x1855007617144192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075484.243915:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617144192/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075484.243942:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075484.243946:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075484.243950:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144192:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075484.243978:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144320:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075484.244004:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075484.244017:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075484/real 1769075484] req@ffff9b03482c4a80 x1855007617144320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075484.244033:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617144320/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075484.244042:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075484.244046:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075484.244049:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144320:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075484.244057:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144448:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075484.244080:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075484.244093:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075484/real 1769075484] req@ffff9b03482c6300 x1855007617144448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075484.244108:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617144448/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075484.244135:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075484.244139:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075484.244143:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144448:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075484.244150:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144576:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075484.244190:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075484.244200:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075484/real 1769075484] req@ffff9b03482c6d80 x1855007617144576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075484.244210:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617144576/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075484.244221:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075484.244224:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075484.244226:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144576:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075484.755341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075485.139318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075485.139370:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075485.779298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075486.163337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075486.163391:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075486.803339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075487.187351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075487.187447:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075487.827323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075488.211311:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075488.211376:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075488.851357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075489.235350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075489.235429:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075489.363318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075489.363334:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075489.363340:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075489.363353:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2478 00000100:00080000:2.0:1769075489.363370:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075489.363382:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075489.363430:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7800 x1855007617144704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075489.363467:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075489.363471:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075489.363473:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075489.363479:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2478 00000100:00080000:2.0:1769075489.363485:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075489.363488:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075489.363497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075489.363509:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617144704:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075489.363511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617144832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075489.363538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075489.363541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075489.363542:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075489.363552:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2478 00000100:00080000:2.0:1769075489.363559:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075489.363566:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075489.363572:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075489.363595:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617144960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075489.363607:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075489.363625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075489.363631:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075489.363634:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075489.363642:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2478 00000100:00080000:2.0:1769075489.363650:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075489.363655:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075489.363660:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075489/real 1769075489] req@ffff9b03482c7800 x1855007617144704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769075489.363666:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075489.363682:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7800 x1855007617144704/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075489.363689:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617145088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075489.363722:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075489.363742:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:2.0:1769075489.363751:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769075489.363752:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617144704:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075489.363764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075489.363769:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075489.363776:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2478 00000100:00080000:2.0:1769075489.363783:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075489.363788:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075489.363794:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144832:192.168.204.160@tcp:38:kworker.0 00000008:00100000:2.0:1769075489.363798:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075489.363815:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617145216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075489.363826:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075489.363852:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075489/real 1769075489] req@ffff9b03482c5180 x1855007617144832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075489.363863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617144832/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075489.363870:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075489.363877:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075489.363880:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144832:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075489.363886:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144960:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075489.363903:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075489.363911:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075489/real 1769075489] req@ffff9b03482c4700 x1855007617144960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075489.363920:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617144960/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075489.363930:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075489.363935:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075489.363940:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617144960:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075489.363949:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145088:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075489.363983:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075489.364011:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075489/real 1769075489] req@ffff9b03482c7100 x1855007617145088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075489.364026:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617145088/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075489.364034:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075489.364038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075489.364040:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145088:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075489.364055:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145216:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075489.364085:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075489.364095:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075489/real 1769075489] req@ffff9b03482c4000 x1855007617145216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075489.364104:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617145216/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075489.364109:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075489.364111:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075489.364113:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145216:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075489.875344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075490.259322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075490.259404:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075490.899335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075491.283341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075491.283435:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075491.923325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075492.307312:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075492.307398:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075492.947341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075493.331322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075493.331371:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075493.971333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075494.355401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075494.355462:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075494.483359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075494.483368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075494.483371:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075494.483380:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2483 00000100:00080000:2.0:1769075494.483388:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075494.483393:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075494.483428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617145344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075494.483453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075494.483456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075494.483457:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075494.483462:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2483 00000100:00080000:2.0:1769075494.483464:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075494.483467:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075494.483474:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075494.483482:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617145472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075494.483490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075494.483493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075494.483494:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075494.483497:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2483 00000100:00080000:2.0:1769075494.483498:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075494.483501:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075494.483504:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075494.483507:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617145344:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075494.483514:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617145600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075494.483522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075494.483524:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075494.483526:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075494.483529:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2483 00000100:00080000:2.0:1769075494.483530:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075494.483533:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075494.483536:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075494.483545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617145728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075494.483551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075494.483554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075494.483555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075494.483558:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2483 00000100:00080000:2.0:1769075494.483560:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075494.483563:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769075494.483568:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769075494.483569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075494.483578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617145856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075494.483607:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075494/real 1769075494] req@ffff9b03482c6a00 x1855007617145344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075494.483620:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617145344/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075494.483627:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075494.483631:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075494.483634:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617145344:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075494.483640:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145472:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075494.483654:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075494.483661:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075494/real 1769075494] req@ffff9b03482c5f80 x1855007617145472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075494.483669:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617145472/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075494.483679:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075494.483681:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075494.483683:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145472:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075494.483700:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145600:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075494.483710:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075494.483716:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075494/real 1769075494] req@ffff9b03482c6680 x1855007617145600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075494.483723:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617145600/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075494.483728:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075494.483730:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075494.483732:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145600:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075494.483735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145728:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075494.483749:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075494.483758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075494/real 1769075494] req@ffff9b03482c5880 x1855007617145728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075494.483771:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617145728/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075494.483780:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075494.483783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075494.483787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145728:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075494.483794:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145856:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075494.483812:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075494.483820:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075494/real 1769075494] req@ffff9b03482c7b80 x1855007617145856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075494.483828:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617145856/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075494.483833:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075494.483835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075494.483837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617145856:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075494.995330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075495.379405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075495.380078:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075496.019357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075496.403366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075496.403502:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075497.043355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075497.427346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075497.427422:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075498.067332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075498.451309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075498.451599:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075499.091322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769075499.475457:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075499.475591:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075499.603385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075499.603396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075499.603401:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075499.603411:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2488 00000100:00080000:3.0:1769075499.603422:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075499.603430:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075499.603468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617145984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075499.603494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075499.603497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075499.603498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075499.603501:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2488 00000100:00080000:3.0:1769075499.603504:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075499.603507:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075499.603513:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075499.603524:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617146112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075499.603535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075499.603540:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075499.603542:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075499.603548:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2488 00000100:00080000:3.0:1769075499.603552:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075499.603561:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617145984:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075499.603561:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075499.603577:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075499.603595:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617146240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075499.603604:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075499.603606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075499.603608:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075499.603610:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2488 00000100:00080000:3.0:1769075499.603613:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075499.603616:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075499.603620:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075499.603627:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617146368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075499.603633:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075499.603635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075499.603636:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075499.603639:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2488 00000100:00080000:3.0:1769075499.603641:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075499.603644:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075499.603646:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075499.603652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617146496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075499.603670:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075499.603727:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075499/real 1769075499] req@ffff9b0352a1aa00 x1855007617145984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075499.603749:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617145984/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075499.603760:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075499.603763:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075499.603767:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617145984:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075499.603774:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146112:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075499.603798:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075499.603807:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075499/real 1769075499] req@ffff9b0352a18700 x1855007617146112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075499.603816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617146112/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075499.603822:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075499.603825:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075499.603827:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146112:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075499.603852:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146240:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075499.603864:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075499.603870:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075499/real 1769075499] req@ffff9b0352a18e00 x1855007617146240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075499.603878:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617146240/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075499.603886:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075499.603890:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075499.603894:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146240:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075499.603931:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146368:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075499.603956:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075499.603972:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075499/real 1769075499] req@ffff9b0352a1a300 x1855007617146368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075499.603987:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617146368/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075499.603993:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075499.603996:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075499.603998:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146368:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075499.604003:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146496:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075499.604016:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075499.604023:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075499/real 1769075499] req@ffff9b0352a19f80 x1855007617146496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075499.604031:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617146496/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075499.604036:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075499.604038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075499.604040:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146496:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075500.115335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075500.499580:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075500.499608:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075501.139316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075501.523325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075501.523382:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075502.163293:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075502.547343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075502.547402:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075503.187361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075503.571344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075503.571397:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075504.211328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075504.595333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075504.595506:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075504.723381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075504.723393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075504.723398:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075504.723409:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2493 00000100:00080000:3.0:1769075504.723420:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075504.723444:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075504.723492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617146624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075504.723529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075504.723534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075504.723536:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075504.723540:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2493 00000100:00080000:3.0:1769075504.723543:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075504.723547:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075504.723557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075504.723571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617146752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075504.723581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075504.723586:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075504.723587:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075504.723591:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2493 00000100:00080000:3.0:1769075504.723594:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075504.723601:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617146624:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075504.723602:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075504.723612:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075504.723622:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617146880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075504.723634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075504.723637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075504.723639:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075504.723643:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2493 00000100:00080000:3.0:1769075504.723645:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075504.723649:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075504.723653:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075504.723664:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617147008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075504.723672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075504.723676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075504.723678:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075504.723682:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2493 00000100:00080000:3.0:1769075504.723685:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075504.723688:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075504.723693:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769075504.723698:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075504.723707:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617147136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075504.723755:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075504/real 1769075504] req@ffff9b0352a1a680 x1855007617146624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075504.723776:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617146624/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075504.723794:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075504.723798:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075504.723803:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617146624:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075504.723810:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146752:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075504.723882:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075504.723902:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075504/real 1769075504] req@ffff9b0352a1b480 x1855007617146752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075504.723922:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617146752/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075504.723936:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075504.723941:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075504.723946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146752:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075504.723979:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146880:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075504.724010:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075504.724020:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075504/real 1769075504] req@ffff9b0352a1ad80 x1855007617146880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075504.724051:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617146880/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075504.724060:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075504.724064:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075504.724077:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617146880:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075504.724084:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147008:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075504.724102:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075504.724111:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075504/real 1769075504] req@ffff9b0352a1b100 x1855007617147008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075504.724123:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617147008/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075504.724141:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075504.724143:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075504.724146:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147008:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075504.724151:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147136:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075504.724180:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075504.724186:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075504/real 1769075504] req@ffff9b0352a18a80 x1855007617147136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075504.724194:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617147136/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075504.724200:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075504.724201:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075504.724203:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147136:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075505.235330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075505.619326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075505.619391:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075506.259367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075506.643388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075506.643518:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075507.283445:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075507.667379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075507.667572:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075508.307360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075508.691339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075508.691391:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075509.331353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075509.715347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075509.715465:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075509.843419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075509.843441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075509.843451:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075509.843472:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2499 00000100:00080000:3.0:1769075509.843491:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075509.843505:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075509.843577:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617147264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075509.843654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075509.843661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075509.843665:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075509.843672:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2499 00000100:00080000:3.0:1769075509.843679:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075509.843689:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075509.843707:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075509.843722:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617147264:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075509.843757:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617147392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075509.843784:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075509.843791:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075509.843795:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075509.843812:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2499 00000100:00080000:3.0:1769075509.843819:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075509.843828:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075509.843836:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075509.843855:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617147520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075509.843875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075509.843879:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075509.843881:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769075509.843892:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075509.843897:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2499 00000100:00080000:3.0:1769075509.843907:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075509.843914:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075509.843922:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075509.843937:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617147648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075509.843949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075509.843953:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075509.843956:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075509.843964:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2499 00000100:00100000:0.0:1769075509.843965:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075509/real 1769075509] req@ffff9b0352a18000 x1855007617147264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075509.843968:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075509.843977:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075509.843988:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075509.844007:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617147776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075509.844011:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617147264/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075509.844049:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075509.844056:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075509.844063:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617147264:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075509.844097:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147392:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075509.844138:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075509.844152:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075509/real 1769075509] req@ffff9b0352a19500 x1855007617147392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075509.844167:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617147392/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075509.844182:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075509.844188:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075509.844194:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147392:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075509.844205:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147520:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075509.844303:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075509.844319:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075509/real 1769075509] req@ffff9b0352a19c00 x1855007617147520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075509.844333:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617147520/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075509.844350:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075509.844354:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075509.844358:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147520:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075509.844365:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147648:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075509.844385:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075509.844395:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075509/real 1769075509] req@ffff9b0352a18380 x1855007617147648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075509.844423:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617147648/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075509.844433:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075509.844436:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075509.844440:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147648:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075509.844446:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147776:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075509.844464:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075509.844475:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075509/real 1769075509] req@ffff9b0352a1b800 x1855007617147776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075509.844497:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617147776/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075509.844506:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075509.844509:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075509.844514:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617147776:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075510.355396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075510.739370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075510.739432:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075511.379385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075511.763328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075511.763392:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075512.403433:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075512.787321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075512.787372:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075513.427359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075513.811368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075513.811452:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075514.451403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075514.835363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075514.835439:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075514.963332:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075514.963341:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075514.963346:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075514.963354:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2504 00000100:00080000:2.0:1769075514.963362:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075514.963367:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075514.963401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617147904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075514.963424:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075514.963428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075514.963429:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075514.963432:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2504 00000100:00080000:2.0:1769075514.963434:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075514.963437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075514.963443:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075514.963451:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617148032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075514.963465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075514.963469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075514.963471:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075514.963475:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2504 00000100:00080000:2.0:1769075514.963478:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075514.963482:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075514.963486:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075514.963498:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617148160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075514.963509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075514.963513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075514.963515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075514.963520:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2504 00000100:00080000:2.0:1769075514.963525:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075514.963529:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075514.963536:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075514.963548:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617148288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075514.963559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075514.963582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769075514.963583:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617147904:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075514.963584:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075514.963593:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2504 00000100:00080000:2.0:1769075514.963607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075514.963612:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075514.963617:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075514.963629:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617148416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075514.963818:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148032:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075514.963856:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148160:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075514.963917:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148288:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075514.963943:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148416:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075515.475326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075515.859390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075515.859459:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075516.499343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075516.883330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075516.883394:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075517.523449:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075517.907384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075517.907473:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769075518.035542:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:3.0:1769075518.035585:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769075518.035618:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075518.035663:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075518.035676:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075518.035688:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769075518.035701:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769075518.035714:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075514/real 1769075518] req@ffff9b03482c5880 x1855007617148288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075518.035742:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617148288/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075518.035754:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075518.035759:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075518.035764:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148288:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075518.035773:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075514/real 1769075518] req@ffff9b03482c6680 x1855007617148416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075518.035786:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617148416/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075518.035794:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075518.035797:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075518.035800:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148416:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075518.035804:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075514/real 1769075518] req@ffff9b03482c5c00 x1855007617147904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075518.035813:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617147904/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075518.035823:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075518.035826:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075518.035828:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617147904:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075518.035834:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075514/real 1769075518] req@ffff9b03482c4e00 x1855007617148032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075518.035845:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617148032/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075518.035854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075518.035858:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075518.035862:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148032:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075518.035869:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075514/real 1769075518] req@ffff9b03482c7b80 x1855007617148160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075518.035883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617148160/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075518.035891:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075518.035895:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075518.035898:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148160:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075518.547310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075518.931382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075518.931482:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075519.571860:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075519.955387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075519.955547:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075520.083363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075520.083381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075520.083390:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075520.083407:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2509 00000100:00080000:2.0:1769075520.083423:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075520.083435:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075520.083501:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617148544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075520.083540:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075520.083546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075520.083548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075520.083553:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2509 00000100:00080000:2.0:1769075520.083593:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075520.083611:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075520.083627:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075520.083652:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617148544:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075520.083656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617148672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075520.083699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075520.083705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075520.083709:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075520.083716:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2509 00000100:00080000:2.0:1769075520.083723:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075520.083731:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075520.083740:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075520.083761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617148800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075520.083770:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075520.083790:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075520.083798:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075520.083803:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075520.083812:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2509 00000100:00080000:2.0:1769075520.083819:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075520.083827:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075520.083838:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075520.083849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075520/real 1769075520] req@ffff9b03482c5f80 x1855007617148544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075520.083863:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617148928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075520.083894:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075520.083901:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617148544/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075520.083904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075520.083912:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075520.083930:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2509 00000100:00080000:1.0:1769075520.083938:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075520.083941:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075520.083955:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:2.0:1769075520.083957:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075520.083973:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617148544:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769075520.083973:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075520.084011:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617149056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075520.084024:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148672:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075520.084100:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075520.084122:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075520/real 1769075520] req@ffff9b03482c6a00 x1855007617148672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075520.084145:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617148672/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075520.084176:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075520.084183:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075520.084189:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148672:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075520.084200:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148800:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075520.084267:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075520.084290:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075520/real 1769075520] req@ffff9b03482c4000 x1855007617148800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075520.084312:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617148800/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075520.084347:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075520.084353:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075520.084358:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148800:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075520.084368:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148928:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075520.084400:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075520.084416:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075520/real 1769075520] req@ffff9b03482c7100 x1855007617148928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075520.084436:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617148928/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075520.084449:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075520.084454:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075520.084459:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617148928:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075520.084488:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149056:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075520.084524:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075520.084542:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075520/real 1769075520] req@ffff9b03482c4700 x1855007617149056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075520.084584:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617149056/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075520.084600:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075520.084606:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075520.084613:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149056:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075520.595417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075520.979383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075520.979522:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075521.619422:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075522.003375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075522.003623:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075522.643584:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075523.027389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075523.027533:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075523.667363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075524.051410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075524.051571:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075524.691380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075525.075334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075525.075534:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075525.203307:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075525.203331:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075525.203337:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075525.203349:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2514 00000100:00080000:2.0:1769075525.203362:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075525.203373:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075525.203415:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617149184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075525.203490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075525.203497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075525.203501:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075525.203520:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2514 00000100:00080000:2.0:1769075525.203526:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075525.203533:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075525.203544:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075525.203561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7800 x1855007617149312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075525.203570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075525.203574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075525.203576:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075525.203579:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2514 00000100:00080000:2.0:1769075525.203583:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075525.203586:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075525.203591:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075525.203598:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617149184:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075525.203606:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617149440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075525.203622:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075525.203625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075525.203627:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075525.203632:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2514 00000100:00080000:2.0:1769075525.203645:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075525.203648:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075525.203652:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075525.203664:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617149568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075525.203672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075525.203676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075525.203679:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075525.203685:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2514 00000100:00080000:2.0:1769075525.203690:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075525.203695:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769075525.203700:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769075525.203711:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075525.203733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617149696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075525.203758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075525/real 1769075525] req@ffff9b03482c5180 x1855007617149184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075525.203779:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617149184/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075525.203790:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075525.203795:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075525.203798:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617149184:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075525.203810:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149312:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075525.203834:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075525.203854:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075525/real 1769075525] req@ffff9b03482c7800 x1855007617149312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075525.203863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7800 x1855007617149312/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075525.203871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075525.203875:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075525.203903:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149312:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075525.203945:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149440:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075525.203977:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075525.203988:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075525/real 1769075525] req@ffff9b03482c6d80 x1855007617149440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075525.204003:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617149440/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075525.204012:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075525.204016:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075525.204020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149440:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075525.204026:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149568:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075525.204064:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075525.204098:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075525/real 1769075525] req@ffff9b03482c6300 x1855007617149568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075525.204151:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617149568/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075525.204164:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075525.204172:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075525.204181:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149568:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075525.204192:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149696:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075525.204249:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075525.204279:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075525/real 1769075525] req@ffff9b03482c4a80 x1855007617149696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075525.204369:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617149696/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075525.204385:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075525.204392:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075525.204398:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149696:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075525.715311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075526.099374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075526.099467:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075526.739340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075527.123348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075527.123404:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075527.763569:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075528.147472:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075528.147554:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075528.787303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075529.171339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075529.171439:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075529.811455:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075530.195438:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075530.195506:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075530.323402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075530.323415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075530.323420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075530.323431:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2519 00000100:00080000:0.0:1769075530.323442:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075530.323450:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075530.323496:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617149824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075530.323520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075530.323524:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075530.323525:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075530.323529:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2519 00000100:00080000:0.0:1769075530.323532:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075530.323535:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075530.323545:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075530.323557:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617149824:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075530.323558:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617149952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075530.323573:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075530.323576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075530.323577:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075530.323580:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2519 00000100:00080000:0.0:1769075530.323583:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075530.323586:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075530.323590:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075530.323598:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617150080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075530.323609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075530.323613:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075530.323614:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075530.323616:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2519 00000100:00080000:0.0:1769075530.323627:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075530.323629:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075530.323632:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075530.323641:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075530.323642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617150208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075530.323652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075530.323655:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075530.323657:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075530.323661:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2519 00000100:00080000:0.0:1769075530.323665:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075530.323668:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075530.323671:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075530.323679:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617150336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075530.323693:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075530/real 1769075530] req@ffff9b0365a39180 x1855007617149824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075530.323710:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617149824/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075530.323720:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075530.323724:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075530.323727:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617149824:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075530.323750:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149952:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075530.323769:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075530.323777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075530/real 1769075530] req@ffff9b0365a3ad80 x1855007617149952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075530.323786:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617149952/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075530.323792:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075530.323794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075530.323796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617149952:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075530.323800:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150080:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075530.323810:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075530.323815:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075530/real 1769075530] req@ffff9b0365a3b100 x1855007617150080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075530.323823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617150080/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075530.323829:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075530.323830:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075530.323832:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150080:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075530.323836:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150208:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075530.323846:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075530.323852:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075530/real 1769075530] req@ffff9b0365a3bb80 x1855007617150208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075530.323864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617150208/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075530.323869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075530.323871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075530.323873:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150208:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075530.323877:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150336:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075530.323886:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075530.323891:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075530/real 1769075530] req@ffff9b0365a39500 x1855007617150336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075530.323898:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617150336/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075530.323904:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075530.323906:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075530.323908:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150336:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075530.835307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075531.219478:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075531.219480:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075531.859402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075532.243333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075532.243428:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075532.883432:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075533.267299:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075533.267480:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075533.907316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075534.291389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075534.291462:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075534.931407:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075535.315294:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075535.315364:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075535.443430:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075535.443445:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075535.443451:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075535.443481:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2524 00000100:00080000:3.0:1769075535.443490:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075535.443498:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075535.443561:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617150464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075535.443590:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075535.443594:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075535.443596:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075535.443601:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2524 00000100:00080000:3.0:1769075535.443605:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075535.443610:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075535.443622:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075535.443657:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617150464:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075535.443657:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617150592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075535.443675:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075535.443679:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075535.443681:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075535.443686:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2524 00000100:00080000:3.0:1769075535.443690:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075535.443711:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075535.443716:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075535.443728:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617150720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075535.443738:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769075535.443746:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075535.443746:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075535.443750:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075535.443756:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2524 00000100:00080000:3.0:1769075535.443761:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075535.443765:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075535.443770:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075535.443779:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617150848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075535.443790:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075535.443795:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075535.443797:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075535.443804:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075535/real 1769075535] req@ffff9b0352a1b800 x1855007617150464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075535.443806:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2524 00000100:00080000:3.0:1769075535.443815:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075535.443825:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075535.443843:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075535.443848:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617150464/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075535.443864:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617150976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075535.443868:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075535.443883:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075535.443905:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617150464:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075535.443938:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150592:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075535.443973:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075535.443992:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075535/real 1769075535] req@ffff9b0352a18380 x1855007617150592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075535.444025:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617150592/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075535.444035:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075535.444039:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075535.444043:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150592:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075535.444049:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150720:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075535.444098:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075535.444111:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075535/real 1769075535] req@ffff9b0352a19c00 x1855007617150720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075535.444122:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617150720/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075535.444135:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075535.444137:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075535.444141:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150720:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075535.444146:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150848:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075535.444161:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075535.444169:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075535/real 1769075535] req@ffff9b0352a19500 x1855007617150848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075535.444178:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617150848/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075535.444185:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075535.444188:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075535.444190:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150848:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075535.444194:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150976:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075535.444206:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075535.444213:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075535/real 1769075535] req@ffff9b0352a18000 x1855007617150976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075535.444222:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617150976/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075535.444228:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075535.444231:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075535.444234:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617150976:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075535.955370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075536.339378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075536.339455:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075536.979396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075537.363425:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075537.363535:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075538.003406:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075538.387337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075538.387382:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075539.027404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075539.411431:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075539.411516:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075540.051360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075540.435418:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075540.435479:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075540.563410:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075540.563427:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075540.563435:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075540.563452:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2529 00000100:00080000:3.0:1769075540.563468:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075540.563480:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075540.563530:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617151104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075540.563570:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075540.563575:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075540.563577:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075540.563583:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2529 00000100:00080000:3.0:1769075540.563588:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075540.563596:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075540.563609:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075540.563627:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617151104:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075540.563630:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617151232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075540.563654:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075540.563659:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075540.563662:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075540.563668:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2529 00000100:00080000:3.0:1769075540.563672:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075540.563677:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075540.563684:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075540.563698:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617151360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075540.563710:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075540.563714:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075540.563716:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075540.563720:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2529 00000100:00080000:3.0:1769075540.563725:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075540.563728:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075540.563735:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075540.563744:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075540.563755:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617151488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075540.563770:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075540.563776:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075540.563779:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075540.563784:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2529 00000100:00080000:3.0:1769075540.563789:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075540.563793:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075540.563801:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075540.563824:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075540/real 1769075540] req@ffff9b0352a18a80 x1855007617151104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075540.563828:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617151616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075540.563906:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617151104/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075540.563924:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075540.563931:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075540.563937:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617151104:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075540.563968:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617151232:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075540.564010:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075540.564034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075540/real 1769075540] req@ffff9b0352a1b100 x1855007617151232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075540.564061:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617151232/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075540.564080:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075540.564088:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075540.564095:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617151232:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075540.564108:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617151360:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075540.564139:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075540.564154:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075540/real 1769075540] req@ffff9b0352a1ad80 x1855007617151360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075540.564170:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617151360/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075540.564180:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075540.564184:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075540.564189:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617151360:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075540.564201:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617151488:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075540.564333:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075540.564359:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075540/real 1769075540] req@ffff9b0352a1b480 x1855007617151488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075540.564385:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617151488/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075540.564396:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075540.564400:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075540.564405:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617151488:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075540.564413:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617151616:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075540.564442:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075540.564455:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075540/real 1769075540] req@ffff9b0352a1a680 x1855007617151616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075540.564471:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617151616/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075540.564480:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075540.564484:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075540.564488:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617151616:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075541.075372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075541.459372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075541.459429:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075542.099411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075542.483306:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075542.483420:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075543.123344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075543.507405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075543.507522:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075544.147374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075544.531354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075544.531471:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075545.171394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075545.555358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075545.555447:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075545.683419:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075545.683432:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075545.683438:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075545.683451:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2534 00000100:00080000:2.0:1769075545.683465:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075545.683473:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075545.683533:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617151744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075545.683577:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075545.683583:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075545.683585:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075545.683590:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2534 00000100:00080000:2.0:1769075545.683594:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075545.683600:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075545.683612:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075545.683628:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617151872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075545.683642:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075545.683648:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075545.683653:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075545.683659:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617151744:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075545.683659:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2534 00000100:00080000:2.0:1769075545.683662:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075545.683670:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075545.683684:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075545.683699:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617152000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075545.683709:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075545.683713:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075545.683715:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075545.683718:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2534 00000100:00080000:2.0:1769075545.683721:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075545.683724:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075545.683728:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075545.683737:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617152128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075545.683744:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075545.683748:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075545.683750:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075545.683754:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2534 00000100:00080000:2.0:1769075545.683758:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075545.683760:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075545.683763:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075545.683775:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617152256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075545.683797:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075545.683873:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075545/real 1769075545] req@ffff9b03482c5500 x1855007617151744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075545.683899:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617151744/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075545.683914:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075545.683920:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075545.683926:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617151744:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075545.683959:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617151872:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075545.683999:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075545.684014:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075545/real 1769075545] req@ffff9b03482c4380 x1855007617151872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075545.684030:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617151872/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075545.684041:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075545.684045:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075545.684049:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617151872:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075545.684065:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152000:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075545.684085:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075545.684095:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075545/real 1769075545] req@ffff9b03482c4a80 x1855007617152000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075545.684109:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617152000/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075545.684119:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075545.684123:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075545.684154:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152000:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075545.684162:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152128:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075545.684197:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075545.684210:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075545/real 1769075545] req@ffff9b03482c6300 x1855007617152128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075545.684267:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617152128/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075545.684278:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075545.684282:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075545.684287:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152128:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075545.684295:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152256:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075545.684319:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075545.684332:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075545/real 1769075545] req@ffff9b03482c6d80 x1855007617152256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075545.684346:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617152256/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075545.684357:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075545.684361:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075545.684365:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152256:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075546.195376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075546.579392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075546.579513:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075547.219404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075547.603432:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075547.603544:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075548.243381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075548.627298:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075548.627381:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075549.267446:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075549.651361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075549.651454:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075550.291328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075550.675325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075550.675471:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075550.803399:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075550.803410:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075550.803414:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075550.803424:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2539 00000100:00080000:3.0:1769075550.803435:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075550.803442:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075550.803482:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617152384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075550.803513:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075550.803517:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075550.803519:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075550.803523:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2539 00000100:00080000:3.0:1769075550.803527:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075550.803530:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075550.803540:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075550.803553:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617152512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075550.803566:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617152384:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075550.803568:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075550.803578:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075550.803582:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075550.803587:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2539 00000100:00080000:3.0:1769075550.803590:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075550.803594:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075550.803599:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075550.803610:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617152640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075550.803618:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075550.803621:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075550.803623:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075550.803627:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2539 00000100:00080000:3.0:1769075550.803633:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075550.803639:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075550.803647:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075550.803649:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075550.803670:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617152768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075550.803687:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075550.803692:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075550.803696:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075550.803699:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075550/real 1769075550] req@ffff9b0352a19f80 x1855007617152384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075550.803707:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2539 00000100:00080000:3.0:1769075550.803717:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075550.803727:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075550.803732:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617152384/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769075550.803738:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769075550.803750:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075550.803758:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075550.803763:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617152896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075550.803767:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617152384:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075550.803802:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152512:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075550.803827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075550.803849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075550/real 1769075550] req@ffff9b0352a1a300 x1855007617152512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075550.803861:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617152512/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075550.803869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075550.803873:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075550.803876:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152512:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075550.803881:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152640:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075550.803896:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075550.803913:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075550/real 1769075550] req@ffff9b0352a18e00 x1855007617152640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075550.803924:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617152640/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075550.803938:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075550.803940:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075550.803943:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152640:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075550.803948:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152768:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075550.803962:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075550.803969:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075550/real 1769075550] req@ffff9b0352a18700 x1855007617152768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075550.803980:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617152768/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075550.803987:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075550.803990:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075550.803993:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152768:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075550.803998:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152896:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075550.804010:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075550.804018:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075550/real 1769075550] req@ffff9b0352a1aa00 x1855007617152896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075550.804028:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617152896/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075550.804035:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075550.804037:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075550.804071:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617152896:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075551.315440:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075551.699300:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075551.699366:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075552.339322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075552.723298:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075552.723406:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075553.363351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075553.747323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075553.747393:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075554.387327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075554.771328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075554.771381:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075555.411372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075555.795353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075555.795555:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075555.923356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075555.923365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075555.923369:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075555.923377:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2545 00000100:00080000:0.0:1769075555.923385:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075555.923398:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075555.923429:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617153024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075555.923452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075555.923455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075555.923456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075555.923459:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2545 00000100:00080000:0.0:1769075555.923462:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075555.923465:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075555.923489:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075555.923502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617153152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075555.923513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075555.923516:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075555.923518:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075555.923521:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2545 00000100:00080000:0.0:1769075555.923524:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075555.923528:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075555.923532:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075555.923550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617153280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075555.923561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075555.923564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075555.923567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075555.923574:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617153024:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075555.923575:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2545 00000100:00080000:0.0:1769075555.923581:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075555.923591:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075555.923596:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075555.923608:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617153408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075555.923616:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075555.923620:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075555.923622:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075555.923625:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2545 00000100:00080000:0.0:1769075555.923630:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075555.923633:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075555.923638:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075555.923647:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617153536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075555.923676:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075555.923740:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075555/real 1769075555] req@ffff9b0365a3aa00 x1855007617153024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075555.923776:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617153024/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075555.923787:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075555.923791:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075555.923796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617153024:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075555.923804:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153152:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075555.923830:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075555.923852:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075555/real 1769075555] req@ffff9b0365a3b480 x1855007617153152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075555.923882:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617153152/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075555.923894:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075555.923898:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075555.923905:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153152:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075555.923936:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153280:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075555.924022:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075555.924032:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075555/real 1769075555] req@ffff9b0365a38e00 x1855007617153280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075555.924040:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617153280/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075555.924046:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075555.924047:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075555.924050:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153280:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075555.924054:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153408:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075555.924067:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075555.924072:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075555/real 1769075555] req@ffff9b0365a39c00 x1855007617153408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075555.924079:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617153408/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075555.924084:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075555.924086:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075555.924088:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153408:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075555.924091:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153536:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075555.924100:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075555.924106:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075555/real 1769075555] req@ffff9b0365a38a80 x1855007617153536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075555.924113:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617153536/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075555.924118:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075555.924120:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075555.924121:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153536:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075556.435383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075556.819386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075556.819482:0:60112:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075557.459372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075557.843366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075557.843447:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075558.483290:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075558.867378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075558.867472:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075559.507346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075559.891309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075559.891364:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075560.531308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075560.915351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075560.915460:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075561.043296:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075561.043305:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075561.043310:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075561.043318:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2550 00000100:00080000:0.0:1769075561.043326:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075561.043333:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075561.043362:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617153664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075561.043389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075561.043392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075561.043393:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075561.043396:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2550 00000100:00080000:0.0:1769075561.043399:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075561.043402:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075561.043409:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075561.043429:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617153792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075561.043441:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617153664:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075561.043444:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075561.043453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075561.043455:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075561.043461:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2550 00000100:00080000:0.0:1769075561.043467:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075561.043471:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075561.043478:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075561.043493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617153920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075561.043503:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075561.043507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075561.043512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075561.043515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075561.043521:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2550 00000100:00080000:0.0:1769075561.043527:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075561.043532:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075561.043539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075561.043556:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617154048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075561.043573:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075561.043578:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075561.043580:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075561.043585:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2550 00000100:00080000:0.0:1769075561.043590:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075561.043594:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075561.043601:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075561.043604:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075561/real 1769075561] req@ffff9b0365a38a80 x1855007617153664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075561.043615:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617154176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075561.043624:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617153664/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075561.043636:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075561.043640:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075561.043643:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617153664:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075561.043663:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153792:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075561.043680:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075561.043688:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075561/real 1769075561] req@ffff9b0365a39c00 x1855007617153792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075561.043697:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617153792/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075561.043797:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075561.043800:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075561.043804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153792:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075561.043809:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153920:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075561.043907:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075561.043919:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075561/real 1769075561] req@ffff9b0365a38e00 x1855007617153920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075561.043928:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617153920/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075561.043933:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075561.043935:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075561.043937:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617153920:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075561.043941:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617154048:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075561.043956:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075561.043962:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075561/real 1769075561] req@ffff9b0365a3b480 x1855007617154048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075561.043970:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617154048/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075561.043975:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075561.043977:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075561.043979:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617154048:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075561.043983:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617154176:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075561.043998:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075561.044004:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075561/real 1769075561] req@ffff9b0365a3aa00 x1855007617154176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075561.044011:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617154176/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075561.044016:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075561.044018:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075561.044020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617154176:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075561.555354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075561.939396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075561.939466:0:28211:0:(ldlm_lockd.c:2188:__ldlm_bl_to_thread()) added 4/4 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1769075561.939567:0:60112:0:(ldlm_lockd.c:2793:ldlm_bl_get_work()) Got 4 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1769075561.939583:0:60112:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff9b0342ad2800 lock: ffff9b035089be00/0xb7d6bf619879c9eb lrc: 2/0,0 mode: CR/CR res: [0x200000405:0x20c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x7fe2323adfd55e7a expref: -99 pid: 68493 timeout: 0 lvb_type: 0 00000002:00100000:2.0:1769075561.939695:0:60112:0:(mdc_request.c:963:mdc_close()) @@@ matched open req@ffff9b0365a3a680 x1855007617062144/t8589944144(8589944144) o101->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 608/696 e 0 to 0 dl 1769074987 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1769075561.939713:0:60112:0:(genops.c:2438:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff9b0342ad2800: modify RPC slot 3 is allocated opc 35, max 7 00000100:00100000:2.0:1769075561.939724:0:60112:0:(client.c:806:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9b03482c7800 x1855007617154432/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00080000:2.0:1769075561.939756:0:60112:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9b03482c7800 x1855007617154432/t0(0) o35->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:23/10 lens 392/2128 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1769075561.939769:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075561.939888:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075562.579304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075562.963476:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075562.964111:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075562.964267:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075563.603389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075563.987381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075563.987466:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075563.987639:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075564.627365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075565.011450:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075565.011579:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075565.011618:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075565.651350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075566.035467:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075566.035469:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075566.035649:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075566.163453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075566.163469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075566.163475:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075566.163489:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2555 00000100:00080000:2.0:1769075566.163504:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075566.163514:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075566.163579:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617154560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075566.163634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075566.163638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075566.163640:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075566.163663:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2555 00000100:00080000:2.0:1769075566.163670:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075566.163676:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075566.163701:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075566.163725:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617154688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075566.163730:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617154560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075566.163766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075566.163771:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075566.163775:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075566.163814:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2555 00000100:00080000:2.0:1769075566.163835:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075566.163841:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075566.163850:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075566.163876:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617154816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075566.163895:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075566.163900:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075566.163902:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075566.163911:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2555 00000100:00080000:2.0:1769075566.163920:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769075566.163948:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075566.163953:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075566.163967:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075566.163996:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617154944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075566.164010:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075566/real 1769075566] req@ffff9b03482c5180 x1855007617154560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075566.164019:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075566.164041:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075566.164045:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075566.164056:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2555 00000100:00080000:2.0:1769075566.164064:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075566.164079:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075566.164097:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075566.164131:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617155072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075566.164155:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617154560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075566.164175:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075566.164183:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075566.164190:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617154560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075566.164246:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617154688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075566.164331:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075566.164346:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075566/real 1769075566] req@ffff9b03482c4700 x1855007617154688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075566.164358:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617154688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075566.164394:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075566.164398:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075566.164402:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617154688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075566.164411:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617154816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075566.164434:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075566.164445:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075566/real 1769075566] req@ffff9b03482c7100 x1855007617154816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075566.164458:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617154816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075566.164475:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075566.164479:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075566.164482:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617154816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075566.164498:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617154944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075566.164515:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075566.164524:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075566/real 1769075566] req@ffff9b03482c4000 x1855007617154944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075566.164537:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617154944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075566.164547:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075566.164553:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075566.164559:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617154944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075566.164570:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075566.164618:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075566.164640:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075566/real 1769075566] req@ffff9b03482c6a00 x1855007617155072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075566.164665:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617155072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075566.164712:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075566.164717:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075566.164723:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075566.675389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075567.059414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075567.059889:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075567.060528:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075567.699378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075568.083384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075568.083392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075568.083484:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075568.723335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075569.107326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075569.107443:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075569.107459:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075569.747353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075570.131315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075570.131328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075570.131654:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075570.771388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075571.155358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075571.155358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075571.155511:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075571.283302:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075571.283311:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075571.283315:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075571.283323:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2560 00000100:00080000:2.0:1769075571.283331:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075571.283337:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075571.283366:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617155200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075571.283389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075571.283392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075571.283393:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075571.283397:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2560 00000100:00080000:2.0:1769075571.283399:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075571.283402:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075571.283408:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075571.283415:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617155328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075571.283421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075571.283424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075571.283425:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075571.283428:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2560 00000100:00080000:2.0:1769075571.283430:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075571.283432:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075571.283435:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075571.283443:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617155456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075571.283449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075571.283451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075571.283453:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075571.283455:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2560 00000100:00080000:2.0:1769075571.283458:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075571.283463:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617155200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075571.283464:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075571.283472:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075571.283480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617155584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075571.283487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075571.283490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075571.283491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075571.283494:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2560 00000100:00080000:2.0:1769075571.283496:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075571.283498:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075571.283501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075571.283508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617155712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075571.283536:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075571.283574:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075571/real 1769075571] req@ffff9b03482c5f80 x1855007617155200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075571.283589:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617155200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075571.283596:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075571.283610:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075571.283613:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617155200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075571.283618:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075571.283634:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075571.283641:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075571/real 1769075571] req@ffff9b03482c7b80 x1855007617155328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075571.283649:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617155328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075571.283655:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075571.283657:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075571.283659:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075571.283664:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075571.283674:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075571.283680:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075571/real 1769075571] req@ffff9b03482c4e00 x1855007617155456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075571.283687:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617155456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075571.283692:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075571.283693:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075571.283708:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075571.283731:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075571.283757:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075571.283764:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075571/real 1769075571] req@ffff9b03482c5c00 x1855007617155584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075571.283772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617155584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075571.283777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075571.283779:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075571.283781:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075571.283784:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075571.283793:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075571.283798:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075571/real 1769075571] req@ffff9b03482c6680 x1855007617155712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075571.283806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617155712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075571.283811:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075571.283813:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075571.283814:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075571.795386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075572.179361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075572.179361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075572.179547:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075572.819420:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075573.203330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075573.203379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075573.203466:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075573.843389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075574.227396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075574.227396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075574.227474:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075574.867359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075575.251295:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075575.251308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075575.251367:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075575.891352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075576.275336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075576.275336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075576.275400:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075576.403314:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075576.403325:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075576.403330:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075576.403339:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2565 00000100:00080000:3.0:1769075576.403349:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075576.403357:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075576.403392:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617155840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075576.403416:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075576.403419:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075576.403421:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075576.403424:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2565 00000100:00080000:3.0:1769075576.403427:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075576.403432:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075576.403448:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075576.403449:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617155840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075576.403468:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617155968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075576.403482:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075576.403487:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075576.403489:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075576.403493:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2565 00000100:00080000:3.0:1769075576.403499:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075576.403503:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075576.403508:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075576.403509:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075576.403521:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617156096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075576.403538:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075576.403540:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075576.403542:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075576.403545:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2565 00000100:00080000:3.0:1769075576.403548:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075576.403551:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075576.403556:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075576.403558:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075576/real 1769075576] req@ffff9b0352a1aa00 x1855007617155840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075576.403566:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617156224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075576.403576:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075576.403580:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617155840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075576.403581:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075576.403584:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075576.403596:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075576.403600:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2565 00000100:00080000:1.0:1769075576.403604:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769075576.403605:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075576.403625:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617155840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075576.403626:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075576.403634:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075576.403649:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617156352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075576.403653:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075576.403691:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075576.403705:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075576/real 1769075576] req@ffff9b0352a18700 x1855007617155968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075576.403719:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617155968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075576.403728:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075576.403732:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075576.403736:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617155968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075576.403743:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075576.403763:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075576.403774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075576/real 1769075576] req@ffff9b0352a18e00 x1855007617156096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075576.403788:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617156096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075576.403793:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075576.403796:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075576.403798:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075576.403802:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075576.403815:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075576.403821:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075576/real 1769075576] req@ffff9b0352a1a300 x1855007617156224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075576.403828:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617156224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075576.403833:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075576.403835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075576.403837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075576.403848:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075576.403858:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075576.403867:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075576/real 1769075576] req@ffff9b0352a19f80 x1855007617156352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075576.403880:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617156352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075576.403894:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075576.403898:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075576.403901:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075576.915357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075577.299339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075577.299365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075577.299494:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075577.939348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075578.323358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075578.323358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075578.323484:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075578.963324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075579.347315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075579.347315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075579.347392:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075579.987436:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075580.371434:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075580.371435:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075580.371523:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075581.011372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075581.395410:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075581.395413:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075581.395480:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075581.523371:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075581.523394:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075581.523399:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075581.523411:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2570 00000100:00080000:3.0:1769075581.523422:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075581.523459:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075581.523506:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617156480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075581.523598:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075581.523603:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075581.523605:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075581.523610:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2570 00000100:00080000:3.0:1769075581.523625:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075581.523630:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075581.523641:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075581.523657:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617156608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075581.523670:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075581.523676:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075581.523681:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075581.523685:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617156480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075581.523693:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2570 00000100:00080000:3.0:1769075581.523703:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075581.523728:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075581.523736:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075581.523755:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617156736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075581.523769:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075581.523774:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075581.523778:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075581.523783:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2570 00000100:00080000:3.0:1769075581.523788:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075581.523792:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075581.523797:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075581.523822:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617156864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075581.523831:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075581.523835:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075581.523837:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075581.523841:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2570 00000100:00080000:3.0:1769075581.523845:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075581.523848:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156608:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075581.523853:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075581.523862:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075581.523884:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617156992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075581.523905:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075581.523948:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075581.523971:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075582.035327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075582.419406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075582.419406:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075582.419550:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075583.059405:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075583.443312:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075583.443313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075583.443467:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075584.083331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075584.467323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075584.467323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075584.467383:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769075584.595629:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769075584.595722:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769075584.595739:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769075584.595777:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769075584.595786:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769075584.595796:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769075584.595806:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769075584.595842:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075581/real 1769075584] req@ffff9b0352a18a80 x1855007617156992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075584.595868:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617156992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075584.595880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075584.595885:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075584.595891:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075584.595899:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075581/real 1769075584] req@ffff9b0352a1b480 x1855007617156608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075584.595913:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617156608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075584.595923:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075584.595927:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075584.595930:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075584.595937:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075581/real 1769075584] req@ffff9b0352a1ad80 x1855007617156736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075584.595949:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617156736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075584.595957:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075584.595961:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075584.595965:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075584.595972:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075581/real 1769075584] req@ffff9b0352a1b100 x1855007617156864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075584.595984:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617156864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075584.595990:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075584.595993:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075584.595997:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617156864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075584.596005:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075581/real 1769075584] req@ffff9b0352a1a680 x1855007617156480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075584.596018:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617156480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075584.596026:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075584.596030:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075584.596033:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617156480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075585.107332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075585.491315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075585.491316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075585.491442:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075586.131468:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075586.515385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075586.515385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075586.515430:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075586.643363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075586.643374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075586.643378:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075586.643386:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2575 00000100:00080000:0.0:1769075586.643394:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075586.643399:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075586.643470:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617157120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075586.643496:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075586.643500:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075586.643502:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075586.643506:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2575 00000100:00080000:0.0:1769075586.643510:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075586.643518:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075586.643530:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075586.643533:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617157120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075586.643549:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617157248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075586.643563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075586.643567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075586.643569:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075586.643574:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2575 00000100:00080000:0.0:1769075586.643595:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075586.643600:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769075586.643606:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769075586.643608:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075586.643624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617157376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075586.643635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075586.643639:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075586.643642:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075586.643646:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2575 00000100:00100000:1.0:1769075586.643652:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075586/real 1769075586] req@ffff9b0365a39500 x1855007617157120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075586.643654:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075586.643663:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075586.643672:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075586.643686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617157504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075586.643691:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617157120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075586.643704:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075586.643711:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075586.643713:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075586.643714:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075586.643725:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2575 00000100:00080000:1.0:1769075586.643725:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:0.0:1769075586.643733:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075586.643736:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617157120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075586.643742:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075586.643750:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075586.643763:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617157632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075586.643770:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617157248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075586.643807:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075586.643822:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075586/real 1769075586] req@ffff9b0365a3bb80 x1855007617157248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075586.643837:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617157248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075586.643847:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075586.643851:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075586.643854:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617157248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075586.643861:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617157376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075586.643883:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075586.643895:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075586/real 1769075586] req@ffff9b0365a3b100 x1855007617157376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075586.643909:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617157376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075586.643928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075586.643932:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075586.643936:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617157376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075586.643943:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617157504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075586.643967:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075586.643979:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075586/real 1769075586] req@ffff9b0365a3ad80 x1855007617157504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075586.644004:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617157504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075586.644013:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075586.644017:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075586.644020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617157504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075586.644038:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617157632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075586.644058:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075586.644069:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075586/real 1769075586] req@ffff9b0365a39180 x1855007617157632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075586.644082:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617157632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075586.644090:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075586.644094:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075586.644098:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617157632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075587.155337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075587.539312:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075587.539313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075587.539386:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075588.179344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075588.563321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075588.563346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075588.563385:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075589.203303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075589.587307:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075589.587307:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075589.587383:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075590.227288:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075590.611281:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075590.611311:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075590.611401:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075591.251435:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075591.635337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075591.635412:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075591.635454:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075591.763388:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075591.763408:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075591.763412:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075591.763421:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2580 00000100:00080000:3.0:1769075591.763430:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075591.763437:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075591.763482:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617157760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075591.763508:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075591.763512:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075591.763514:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075591.763518:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2580 00000100:00080000:3.0:1769075591.763521:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075591.763526:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075591.763535:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075591.763551:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617157760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075591.763552:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617157888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075591.763570:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075591.763573:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075591.763575:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075591.763579:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2580 00000100:00080000:3.0:1769075591.763582:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075591.763585:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075591.763590:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075591.763601:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617158016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075591.763609:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075591.763612:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075591.763614:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075591.763617:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2580 00000100:00080000:3.0:1769075591.763620:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075591.763622:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769075591.763629:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769075591.763629:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075591.763643:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617158144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075591.763653:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075591.763656:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075591.763657:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075591.763660:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2580 00000100:00080000:3.0:1769075591.763663:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075591.763666:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075591.763670:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075591.763685:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075591/real 1769075591] req@ffff9b0352a18000 x1855007617157760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075591.763685:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617158272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075591.763737:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617157760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075591.763777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075591.763781:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075591.763785:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617157760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075591.763807:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617157888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075591.763831:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075591.763854:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075591/real 1769075591] req@ffff9b0352a19500 x1855007617157888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075591.763871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617157888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075591.763882:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075591.763887:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075591.763892:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617157888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075591.763900:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075591.763935:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075591.763958:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075591/real 1769075591] req@ffff9b0352a19c00 x1855007617158016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075591.763969:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617158016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075591.763976:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075591.763979:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075591.763982:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075591.763987:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075591.764004:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075591.764013:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075591/real 1769075591] req@ffff9b0352a18380 x1855007617158144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075591.764022:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617158144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075591.764028:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075591.764030:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075591.764033:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075591.764037:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075591.764049:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075591.764055:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075591/real 1769075591] req@ffff9b0352a1b800 x1855007617158272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075591.764065:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617158272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075591.764071:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075591.764073:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075591.764075:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075592.275314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075592.659313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075592.659349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075592.659400:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075593.299379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075593.683373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075593.683402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075593.683457:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075594.323373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075594.707292:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075594.707292:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075594.707378:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075595.347414:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075595.731406:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075595.731409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075595.731453:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075596.371339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075596.755324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075596.755324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075596.755429:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075596.883373:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075596.883385:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075596.883389:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075596.883401:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2586 00000100:00080000:3.0:1769075596.883411:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075596.883420:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075596.883457:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617158400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075596.883485:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075596.883489:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075596.883493:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075596.883498:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2586 00000100:00080000:3.0:1769075596.883503:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075596.883510:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617158400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075596.883510:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075596.883525:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075596.883542:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617158528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075596.883553:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075596.883557:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075596.883560:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075596.883565:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2586 00000100:00080000:3.0:1769075596.883570:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769075596.883573:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075596.883579:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075596.883587:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075596.883600:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617158656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075596.883608:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075596.883611:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075596.883613:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075596.883618:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075596/real 1769075596] req@ffff9b0352a1b800 x1855007617158400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075596.883618:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2586 00000100:00100000:0.0:1769075596.883651:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617158400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075596.883664:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075596.883668:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075596.883671:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617158400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075596.883687:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075596.883693:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158528:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075596.883695:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075596.883704:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769075596.883719:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075596.883724:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617158784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075596.883734:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075596/real 1769075596] req@ffff9b0352a18380 x1855007617158528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075596.883741:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075596.883747:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769075596.883749:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617158528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075596.883752:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075596.883762:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075596.883764:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2586 00000100:00080000:0.0:1769075596.883767:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075596.883772:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158528:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075596.883773:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075596.883779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158656:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075596.883783:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075596.883790:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769075596.883800:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075596.883805:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617158912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075596.883810:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075596/real 1769075596] req@ffff9b0352a19c00 x1855007617158656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075596.883828:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617158656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075596.883834:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075596.883837:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075596.883840:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075596.883850:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075596.883861:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075596.883867:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075596/real 1769075596] req@ffff9b0352a19500 x1855007617158784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075596.883875:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617158784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075596.883885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075596.883887:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075596.883889:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075596.883892:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075596.883901:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075596.883906:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075596/real 1769075596] req@ffff9b0352a18000 x1855007617158912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075596.883913:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617158912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075596.883918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075596.883920:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075596.883921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617158912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075597.395354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075597.779352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075597.779352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075597.779450:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075598.419321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075598.803301:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075598.803301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075598.803343:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075599.443369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075599.827303:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075599.827312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075599.827389:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075600.467313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075600.851426:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075600.851426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075600.851525:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075601.491335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075601.875410:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075601.875410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075601.875471:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075602.003349:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075602.003359:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075602.003363:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075602.003371:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2591 00000100:00080000:0.0:1769075602.003378:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075602.003384:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075602.003431:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617159040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075602.003462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075602.003464:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075602.003466:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075602.003469:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2591 00000100:00080000:0.0:1769075602.003471:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075602.003474:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075602.003481:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075602.003490:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617159168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075602.003498:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075602.003503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075602.003504:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075602.003504:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617159040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075602.003512:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2591 00000100:00080000:0.0:1769075602.003515:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075602.003518:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075602.003521:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075602.003529:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617159296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075602.003535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075602.003537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075602.003538:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075602.003541:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2591 00000100:00080000:0.0:1769075602.003544:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075602.003547:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075602.003551:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075602.003558:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617159424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075602.003564:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075602.003566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075602.003567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075602.003570:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2591 00000100:00080000:0.0:1769075602.003572:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075602.003574:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075602.003577:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075602.003583:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075602.003587:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617159552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075602.003644:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9b0365a39180 x1855007617159040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075602.003660:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617159040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075602.003669:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075602.003674:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075602.003678:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617159040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075602.003702:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075602.003727:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075602.003741:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9b0365a3ad80 x1855007617159168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075602.003752:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617159168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075602.003758:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075602.003760:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075602.003763:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075602.003767:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075602.003782:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075602.003788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9b0365a3b100 x1855007617159296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075602.003795:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617159296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075602.003801:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075602.003803:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075602.003805:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075602.003808:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075602.003818:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075602.003823:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9b0365a3bb80 x1855007617159424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075602.003830:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617159424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075602.003835:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075602.003837:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075602.003838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075602.003842:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075602.003850:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075602.003856:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075602/real 1769075602] req@ffff9b0365a39500 x1855007617159552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075602.003863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617159552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075602.003869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075602.003870:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075602.003873:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075602.515357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075602.899366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075602.899443:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075602.899529:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075603.539340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075603.923362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075603.923555:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075603.923588:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075604.563345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075604.947368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075604.947419:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075604.947423:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075605.587362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075605.971328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075605.971328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075605.971343:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075606.611373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075606.995352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075606.995357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075606.995425:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075607.123418:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075607.123431:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075607.123436:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075607.123446:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2596 00000100:00080000:3.0:1769075607.123458:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075607.123479:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075607.123521:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617159680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075607.123569:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075607.123574:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075607.123577:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075607.123581:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2596 00000100:00080000:3.0:1769075607.123585:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075607.123590:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075607.123600:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075607.123613:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617159808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075607.123627:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075607.123633:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1769075607.123635:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617159680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075607.123635:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075607.123645:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2596 00000100:00080000:3.0:1769075607.123654:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075607.123658:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075607.123662:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075607.123674:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617159936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075607.123683:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075607.123686:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075607.123689:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075607.123693:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2596 00000100:00080000:3.0:1769075607.123697:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075607.123701:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075607.123706:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075607.123716:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617160064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075607.123725:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075607.123729:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075607.123731:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769075607.123740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075607.123742:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2596 00000100:00080000:3.0:1769075607.123750:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075607.123753:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075607.123758:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075607.123770:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617160192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075607.123816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075607/real 1769075607] req@ffff9b0352a1a680 x1855007617159680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075607.123839:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617159680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075607.123850:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075607.123855:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075607.123860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617159680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075607.123885:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075607.123914:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075607.123927:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075607/real 1769075607] req@ffff9b0352a1b100 x1855007617159808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075607.123939:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617159808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075607.123974:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075607.123978:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075607.123982:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075607.123989:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075607.124006:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075607.124016:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075607/real 1769075607] req@ffff9b0352a1ad80 x1855007617159936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075607.124028:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617159936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075607.124047:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075607.124050:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075607.124054:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617159936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075607.124060:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075607.124075:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075607.124087:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075607/real 1769075607] req@ffff9b0352a1b480 x1855007617160064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075607.124178:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617160064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075607.124190:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075607.124195:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075607.124201:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075607.124210:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075607.124275:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075607.124294:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075607/real 1769075607] req@ffff9b0352a18a80 x1855007617160192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075607.124312:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617160192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075607.124325:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075607.124330:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075607.124336:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075607.635337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075608.019316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075608.019319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075608.019397:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075608.659327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075609.043298:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075609.043304:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075609.043373:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075609.683345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075610.067331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075610.067333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075610.067429:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075610.707307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075611.091321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075611.091325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075611.091460:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075611.731387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075612.115351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075612.115351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075612.115438:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075612.243354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075612.243369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075612.243376:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075612.243390:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2601 00000100:00080000:2.0:1769075612.243404:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075612.243414:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075612.243465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617160320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075612.243500:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075612.243505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075612.243509:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075612.243515:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2601 00000100:00080000:2.0:1769075612.243520:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075612.243526:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075612.243536:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617160320:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769075612.243540:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075612.243573:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617160448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075612.243589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075612.243594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075612.243596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075612.243602:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2601 00000100:00080000:2.0:1769075612.243607:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769075612.243612:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075612.243614:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075612.243624:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075612.243639:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617160576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075612.243652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075612.243675:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075612.243677:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075612.243682:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2601 00000100:00080000:2.0:1769075612.243688:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075612.243692:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075612/real 1769075612] req@ffff9b03482c5880 x1855007617160320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075612.243696:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075612.243705:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075612.243720:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617160704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075612.243725:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617160320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075612.243739:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075612.243746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075612.243747:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075612.243749:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075612.243755:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:2.0:1769075612.243758:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2601 00000100:00100000:1.0:1769075612.243762:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617160320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075612.243765:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075612.243772:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075612.243777:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075612.243789:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617160832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075612.243792:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075612.243823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075612.243843:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075612/real 1769075612] req@ffff9b03482c6680 x1855007617160448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075612.243853:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617160448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075612.243860:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075612.243862:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075612.243864:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075612.243869:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075612.243882:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075612.243888:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075612/real 1769075612] req@ffff9b03482c5c00 x1855007617160576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075612.243896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617160576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075612.243907:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075612.243909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075612.243911:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075612.243915:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075612.243926:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075612.243948:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075612/real 1769075612] req@ffff9b03482c4e00 x1855007617160704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075612.243957:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617160704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075612.243964:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075612.243966:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075612.243968:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075612.243979:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075612.243991:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075612.243998:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075612/real 1769075612] req@ffff9b03482c7b80 x1855007617160832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075612.244007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617160832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075612.244085:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075612.244088:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075612.244107:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617160832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075612.755333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075613.139356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075613.139356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075613.139463:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075613.779362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075614.163331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075614.163344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075614.163437:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075614.803354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075615.187360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075615.187376:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075615.187514:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075615.827364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075616.211339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075616.211342:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075616.211343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075616.851321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075617.235321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075617.235380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075617.235388:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075617.363315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075617.363332:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075617.363338:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075617.363353:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2606 00000100:00080000:2.0:1769075617.363367:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075617.363374:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075617.363418:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617160960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075617.363451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075617.363454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075617.363455:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075617.363459:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2606 00000100:00080000:2.0:1769075617.363463:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075617.363466:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075617.363479:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075617.363497:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617160960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075617.363499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617161088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075617.363523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075617.363528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075617.363530:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075617.363537:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2606 00000100:00080000:2.0:1769075617.363541:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075617.363547:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075617.363552:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075617.363565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617161216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075617.363576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769075617.363577:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075617.363585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075617.363587:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075617.363591:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2606 00000100:00080000:2.0:1769075617.363596:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075617.363599:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075617.363604:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075617.363612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617161344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075617.363620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075617.363623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075617.363625:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075617.363630:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2606 00000100:00080000:2.0:1769075617.363634:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075617.363640:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075617.363645:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075617/real 1769075617] req@ffff9b03482c5f80 x1855007617160960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769075617.363646:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075617.363668:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617161472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075617.363682:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617160960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075617.363692:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075617.363697:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075617.363701:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617160960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075617.363723:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075617.363743:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075617.363752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075617/real 1769075617] req@ffff9b03482c6a00 x1855007617161088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075617.363760:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617161088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075617.363766:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075617.363768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075617.363771:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075617.363776:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075617.363786:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075617.363792:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075617/real 1769075617] req@ffff9b03482c4000 x1855007617161216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075617.363799:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617161216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075617.363804:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075617.363806:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075617.363808:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075617.363812:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075617.363822:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075617.363827:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075617/real 1769075617] req@ffff9b03482c7100 x1855007617161344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075617.363834:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617161344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075617.363839:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075617.363841:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075617.363843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075617.363846:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075617.363855:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075617.363862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075617/real 1769075617] req@ffff9b03482c4700 x1855007617161472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075617.363869:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617161472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075617.363874:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075617.363876:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075617.363878:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075617.875318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075618.259313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075618.259321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075618.259371:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075618.899332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075619.283346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075619.283382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075619.283458:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075619.923378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075620.307350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075620.307350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075620.307391:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075620.947314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075621.331348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075621.331348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075621.331373:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075621.971328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075622.355313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075622.355318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075622.355444:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075622.483430:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075622.483442:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075622.483447:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075622.483458:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2611 00000100:00080000:3.0:1769075622.483478:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075622.483496:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075622.483541:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617161600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075622.483577:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075622.483580:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075622.483582:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075622.483585:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2611 00000100:00080000:3.0:1769075622.483587:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075622.483590:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075622.483599:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075622.483608:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617161728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075622.483617:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617161600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075622.483619:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075622.483626:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075622.483629:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075622.483641:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2611 00000100:00080000:3.0:1769075622.483643:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075622.483645:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075622.483648:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075622.483658:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617161856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075622.483664:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075622.483666:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075622.483667:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075622.483670:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2611 00000100:00080000:3.0:1769075622.483673:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075622.483676:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075622.483680:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075622.483687:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617161984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075622.483692:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075622.483705:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075622.483707:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075622.483710:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2611 00000100:00080000:3.0:1769075622.483713:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075622.483715:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075622.483719:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075622.483724:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075622.483729:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617162112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075622.483784:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075622/real 1769075622] req@ffff9b0352a19f80 x1855007617161600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075622.483800:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617161600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075622.483809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075622.483813:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075622.483816:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617161600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075622.483878:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075622.483907:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075622.483917:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075622/real 1769075622] req@ffff9b0352a1a300 x1855007617161728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075622.483926:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617161728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075622.483932:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075622.483935:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075622.483937:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075622.483941:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075622.483952:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075622.483958:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075622/real 1769075622] req@ffff9b0352a18e00 x1855007617161856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075622.483966:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617161856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075622.483971:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075622.483973:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075622.483974:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075622.483978:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075622.483988:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075622.483994:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075622/real 1769075622] req@ffff9b0352a18700 x1855007617161984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075622.484001:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617161984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075622.484007:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075622.484009:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075622.484011:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617161984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075622.484014:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617162112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075622.484023:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075622.484028:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075622/real 1769075622] req@ffff9b0352a1aa00 x1855007617162112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075622.484034:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617162112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075622.484039:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075622.484041:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075622.484043:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617162112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075622.995316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075623.379342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075623.379342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075623.379431:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075624.019316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075624.403332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075624.403332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075624.403415:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075625.043364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075625.427338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075625.427338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075625.427387:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075626.067315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075626.451318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075626.451338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075626.451363:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075627.091366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075627.475323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075627.475358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075627.475391:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075627.603351:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075627.603361:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075627.603366:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075627.603375:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2616 00000100:00080000:3.0:1769075627.603384:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075627.603390:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075627.603443:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617162240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075627.603471:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075627.603474:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075627.603476:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075627.603479:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2616 00000100:00080000:3.0:1769075627.603482:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075627.603486:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075627.603495:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075627.603507:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617162368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075627.603512:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617162240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075627.603524:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075627.603527:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075627.603529:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075627.603532:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2616 00000100:00080000:3.0:1769075627.603536:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075627.603539:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075627.603542:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075627.603553:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617162496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075627.603561:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075627.603564:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075627.603565:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075627.603568:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2616 00000100:00080000:3.0:1769075627.603572:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075627.603575:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075627.603579:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075627.603585:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075627.603590:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617162624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075627.603600:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075627.603603:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075627.603605:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075627.603609:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2616 00000100:00080000:3.0:1769075627.603611:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075627.603614:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075627.603619:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075627.603629:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075627/real 1769075627] req@ffff9b0352a1aa00 x1855007617162240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075627.603631:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617162752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075627.603693:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617162240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075627.603705:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075627.603710:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075627.603714:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617162240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075627.603735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617162368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075627.603780:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075627.603809:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075627/real 1769075627] req@ffff9b0352a18700 x1855007617162368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075627.603819:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617162368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075627.603826:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075627.603828:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075627.603831:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617162368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075627.603836:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617162496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075627.603850:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075627.603857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075627/real 1769075627] req@ffff9b0352a18e00 x1855007617162496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075627.603866:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617162496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075627.603877:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075627.603879:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075627.603881:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617162496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075627.603886:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617162624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075627.603897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075627.603903:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075627/real 1769075627] req@ffff9b0352a1a300 x1855007617162624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075627.603911:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617162624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075627.603917:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075627.603919:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075627.603921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617162624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075627.603924:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617162752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075627.603934:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075627.603940:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075627/real 1769075627] req@ffff9b0352a19f80 x1855007617162752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075627.603948:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617162752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075627.603953:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075627.603955:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075627.603959:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617162752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075628.115339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075628.499593:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075628.499625:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075628.499652:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075629.139344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075629.523350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075629.523353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075629.523517:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075630.163311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075630.547355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075630.547426:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075630.547544:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075631.187317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769075631.571732:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075631.571735:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075631.571735:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075632.211343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769075632.595425:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075632.595555:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075632.595666:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769075632.723328:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075632.723338:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075632.723342:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075632.723351:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2621 00000100:00080000:3.0:1769075632.723359:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075632.723366:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075632.723397:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617162880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075632.723424:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075632.723428:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075632.723430:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075632.723434:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2621 00000100:00080000:3.0:1769075632.723437:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075632.723440:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075632.723450:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075632.723469:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617162880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075632.723471:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617163008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075632.723495:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075632.723500:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075632.723503:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075632.723511:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2621 00000100:00080000:3.0:1769075632.723517:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075632.723522:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075632.723528:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075632.723545:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617163136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075632.723558:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075632.723562:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075632.723565:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075632.723573:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2621 00000800:00000100:2.0:1769075632.723580:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075632.723586:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075632.723599:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075632.723606:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075632.723622:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617163264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075632.723634:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075632.723637:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075632.723639:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769075632.723644:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075632/real 1769075632] req@ffff9b0352a18a80 x1855007617162880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075632.723683:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617162880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075632.723697:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075632.723703:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075632.723708:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617162880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075632.723719:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2621 00000100:00080000:3.0:1769075632.723724:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075632.723728:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075632.723732:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075632.723737:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075632.723747:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617163392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769075632.723773:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075632.723790:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075632/real 1769075632] req@ffff9b0352a1b480 x1855007617163008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075632.723805:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617163008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075632.723815:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075632.723819:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075632.723822:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075632.723827:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075632.723845:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075632.723853:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075632/real 1769075632] req@ffff9b0352a1ad80 x1855007617163136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075632.723861:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617163136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075632.723867:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075632.723869:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075632.723872:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075632.723876:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075632.723889:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075632.723896:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075632/real 1769075632] req@ffff9b0352a1b100 x1855007617163264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075632.723909:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617163264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075632.723914:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075632.723916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075632.723918:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075632.723932:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075632.723945:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075632.723952:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075632/real 1769075632] req@ffff9b0352a1a680 x1855007617163392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075632.723968:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617163392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075632.723977:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075632.723981:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075632.723984:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075633.235310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075633.619331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075633.619414:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075633.619474:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075634.259337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075634.643337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075634.643360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075634.643405:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075635.283319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075635.667351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075635.667351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075635.667418:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075636.307337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075636.691362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075636.691376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075636.691446:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075637.331318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075637.715306:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075637.715341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075637.715441:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075637.843334:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075637.843346:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075637.843351:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075637.843368:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2627 00000100:00080000:3.0:1769075637.843379:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075637.843387:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075637.843436:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617163520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075637.843462:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075637.843465:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075637.843467:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075637.843470:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2627 00000100:00080000:3.0:1769075637.843473:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075637.843475:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075637.843484:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075637.843495:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617163648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075637.843505:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075637.843508:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075637.843509:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075637.843514:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2627 00000100:00080000:3.0:1769075637.843517:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075637.843523:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075637.843525:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617163520:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769075637.843529:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075637.843545:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617163776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075637.843555:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075637.843558:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075637.843559:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075637.843564:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2627 00000100:00080000:3.0:1769075637.843566:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075637.843569:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075637.843574:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075637.843594:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617163904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075637.843601:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075637.843604:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075637.843605:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075637.843608:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2627 00000100:00080000:3.0:1769075637.843610:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075637.843613:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075637.843616:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075637.843626:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075637.843627:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617164032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075637.843677:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075637/real 1769075637] req@ffff9b0352a18000 x1855007617163520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075637.843699:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617163520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075637.843712:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075637.843718:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075637.843722:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617163520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075637.843746:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075637.843767:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075637.843776:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075637/real 1769075637] req@ffff9b0352a19500 x1855007617163648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075637.843784:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617163648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075637.843790:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075637.843793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075637.843795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075637.843799:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075637.843810:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075637.843815:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075637/real 1769075637] req@ffff9b0352a19c00 x1855007617163776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075637.843823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617163776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075637.843828:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075637.843840:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075637.843845:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075637.843849:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075637.843861:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075637.843868:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075637/real 1769075637] req@ffff9b0352a18380 x1855007617163904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075637.843875:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617163904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075637.843880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075637.843883:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075637.843885:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617163904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075637.843888:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075637.843897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075637.843903:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075637/real 1769075637] req@ffff9b0352a1b800 x1855007617164032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075637.843910:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617164032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075637.843920:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075637.843922:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075637.843924:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075638.355286:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075638.739328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075638.739346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075638.739373:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075639.379357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075639.763329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075639.763330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075639.763474:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075640.403321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075640.787338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075640.787338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075640.787403:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075641.427321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075641.811308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075641.811310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075641.811365:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075642.451382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075642.835334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075642.835366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075642.835423:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075642.963374:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075642.963384:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075642.963389:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075642.963399:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2632 00000100:00080000:2.0:1769075642.963410:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075642.963418:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075642.963456:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617164160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075642.963506:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075642.963511:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075642.963513:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075642.963518:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2632 00000100:00080000:2.0:1769075642.963524:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075642.963529:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075642.963539:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075642.963553:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617164288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075642.963562:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617164160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075642.963582:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075642.963587:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075642.963589:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075642.963598:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2632 00000100:00080000:2.0:1769075642.963602:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075642.963607:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075642.963614:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075642.963630:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617164416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075642.963642:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075642.963646:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075642.963648:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075642.963654:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2632 00000800:00000100:1.0:1769075642.963659:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075642.963663:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075642.963671:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075642.963678:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075642.963693:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617164544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075642.963707:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075642/real 1769075642] req@ffff9b03482c5180 x1855007617164160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075642.963728:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075642.963733:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075642.963735:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075642.963742:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2632 00000100:00080000:2.0:1769075642.963747:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075642.963748:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617164160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075642.963760:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:1.0:1769075642.963765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769075642.963770:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769075642.963773:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075642.963780:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617164160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075642.963794:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617164672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075642.963815:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075642.963860:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075642.963875:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075642/real 1769075642] req@ffff9b03482c6d80 x1855007617164288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075642.963885:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617164288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075642.963891:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075642.963894:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075642.963896:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075642.963901:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075642.963917:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075642.963923:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075642/real 1769075642] req@ffff9b03482c6300 x1855007617164416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075642.963931:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617164416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075642.963936:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075642.963939:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075642.963941:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075642.963945:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075642.963956:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075642.963961:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075642/real 1769075642] req@ffff9b03482c4a80 x1855007617164544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075642.963973:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617164544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075642.963979:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075642.963981:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075642.963982:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075642.963986:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075642.963995:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075642.964001:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075642/real 1769075642] req@ffff9b03482c4380 x1855007617164672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075642.964007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617164672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075642.964012:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075642.964014:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075642.964016:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075643.475372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075643.859306:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075643.859306:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075643.859376:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075644.499384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075644.883336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075644.883367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075644.883447:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075645.523328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075645.907315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075645.907324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075645.907376:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075646.547398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075646.931306:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075646.931306:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075646.931378:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075647.571330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075647.955370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075647.955372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075647.955476:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075648.083388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075648.083402:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075648.083437:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075648.083451:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2637 00000100:00080000:3.0:1769075648.083463:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075648.083472:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075648.083516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617164800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075648.083548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075648.083551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075648.083553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075648.083557:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2637 00000100:00080000:3.0:1769075648.083562:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075648.083567:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075648.083576:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075648.083589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617164928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075648.083600:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617164800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075648.083612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075648.083616:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075648.083618:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075648.083622:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2637 00000100:00080000:3.0:1769075648.083625:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075648.083630:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075648.083636:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075648.083649:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617165056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075648.083677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075648.083681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075648.083683:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075648.083687:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2637 00000100:00080000:3.0:1769075648.083691:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075648.083696:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075648.083702:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075648.083715:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617165184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075648.083726:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075648.083730:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075648.083732:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075648.083737:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2637 00000100:00080000:3.0:1769075648.083740:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075648.083744:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075648.083751:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075648.083761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617165312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075648.083785:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075648.083833:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075648.083863:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075648.083893:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075648.595339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075648.979295:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075648.979366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075648.979376:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075649.619303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075650.003312:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075650.003320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075650.003396:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075650.643325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075651.027374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075651.027375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075651.027456:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075651.155598:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769075651.155659:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075651.155682:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769075651.155730:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769075651.155745:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769075651.155757:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769075651.155767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075648/real 1769075651] req@ffff9b0352a18380 x1855007617164928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1769075651.155770:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769075651.155803:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617164928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075651.155815:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075651.155820:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075651.155824:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617164928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075651.155832:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075648/real 1769075651] req@ffff9b0352a19c00 x1855007617165056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075651.155846:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617165056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075651.155855:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075651.155859:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075651.155864:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075651.155872:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075648/real 1769075651] req@ffff9b0352a19500 x1855007617165184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075651.155896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617165184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075651.155903:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075651.155906:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075651.155908:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075651.155913:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075648/real 1769075651] req@ffff9b0352a18000 x1855007617165312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075651.155925:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617165312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075651.155935:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075651.155939:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075651.155942:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075651.155952:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075648/real 1769075651] req@ffff9b0352a1b800 x1855007617164800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075651.155972:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617164800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075651.155982:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075651.155986:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075651.155989:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617164800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075651.667371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075652.051323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075652.051400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075652.051431:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075652.691388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075653.075364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075653.075364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075653.075420:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075653.203340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075653.203352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075653.203357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075653.203368:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2642 00000100:00080000:3.0:1769075653.203377:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075653.203385:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075653.203420:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617165440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075653.203446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075653.203449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075653.203451:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075653.203456:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2642 00000100:00080000:3.0:1769075653.203460:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075653.203464:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075653.203473:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075653.203486:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617165568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075653.203506:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075653.203510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075653.203511:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075653.203515:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2642 00000100:00080000:3.0:1769075653.203518:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075653.203522:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075653.203526:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075653.203535:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617165440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075653.203538:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617165696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075653.203549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075653.203552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075653.203553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075653.203556:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2642 00000100:00080000:3.0:1769075653.203559:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075653.203561:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075653.203564:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075653.203586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617165824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075653.203596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075653.203599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075653.203601:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075653.203605:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2642 00000100:00080000:3.0:1769075653.203610:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075653.203616:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769075653.203618:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769075653.203624:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075653.203639:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617165952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075653.203673:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075653/real 1769075653] req@ffff9b0352a1a680 x1855007617165440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075653.203694:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617165440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075653.203704:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075653.203708:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075653.203712:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617165440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075653.203721:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075653.203744:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075653.203758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075653/real 1769075653] req@ffff9b0352a1b100 x1855007617165568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075653.203771:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617165568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075653.203780:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075653.203784:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075653.203787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075653.203823:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075653.203844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075653.203857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075653/real 1769075653] req@ffff9b0352a1ad80 x1855007617165696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075653.203871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617165696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075653.203881:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075653.203884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075653.203888:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075653.203895:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075653.203916:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075653.203929:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075653/real 1769075653] req@ffff9b0352a1b480 x1855007617165824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075653.203941:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617165824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075653.203949:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075653.203952:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075653.203955:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075653.203961:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075653.203981:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075653.203991:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075653/real 1769075653] req@ffff9b0352a18a80 x1855007617165952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075653.204001:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617165952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075653.204015:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075653.204017:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075653.204020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617165952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075653.716272:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075654.099419:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075654.099424:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075654.099486:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075654.739310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075655.123357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075655.123367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075655.123433:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075655.763304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075656.147379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075656.147379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075656.147477:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075656.787309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075657.171340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075657.171358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075657.171390:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075657.811321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075658.195330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075658.195330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075658.195412:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075658.323427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075658.323442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075658.323449:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075658.323462:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2647 00000100:00080000:0.0:1769075658.323474:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075658.323512:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075658.323554:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617166080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075658.323589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075658.323593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075658.323595:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075658.323601:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2647 00000100:00080000:0.0:1769075658.323605:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075658.323610:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075658.323620:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075658.323633:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617166208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075658.323644:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075658.323648:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075658.323650:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075658.323654:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2647 00000100:00080000:0.0:1769075658.323656:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075658.323660:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075658.323665:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075658.323675:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617166336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075658.323694:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075658.323698:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075658.323700:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075658.323704:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2647 00000100:00100000:1.0:1769075658.323731:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617166080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075658.323732:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075658.323739:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075658.323747:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075658.323759:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617166464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075658.323767:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075658.323770:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075658.323772:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075658.323774:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2647 00000100:00080000:0.0:1769075658.323777:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075658.323779:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075658.323783:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075658.323792:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617166592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075658.323821:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075658.323859:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075658/real 1769075658] req@ffff9b0365a3aa00 x1855007617166080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075658.323875:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617166080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075658.323884:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075658.323888:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075658.323892:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617166080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075658.323897:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075658.323914:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075658.323923:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075658/real 1769075658] req@ffff9b0365a3b480 x1855007617166208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075658.323932:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617166208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075658.323939:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075658.323942:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075658.323944:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075658.323948:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075658.323959:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075658.323966:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075658/real 1769075658] req@ffff9b0365a38e00 x1855007617166336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075658.323974:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617166336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075658.323981:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075658.323983:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075658.323985:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075658.324005:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075658.324018:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075658.324025:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075658/real 1769075658] req@ffff9b0365a39c00 x1855007617166464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075658.324034:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617166464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075658.324039:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075658.324041:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075658.324044:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075658.324049:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075658.324065:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075658.324087:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075658/real 1769075658] req@ffff9b0365a38a80 x1855007617166592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075658.324102:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617166592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075658.324112:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075658.324116:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075658.324120:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075658.835348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075659.219352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075659.219356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075659.219421:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075659.859320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075660.243294:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075660.243361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075660.243404:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075660.883328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075661.267327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075661.267327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075661.267400:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075661.907334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075662.291324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075662.291341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075662.291391:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075662.931320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075663.315327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075663.315336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075663.315408:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075663.443343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075663.443352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075663.443357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075663.443366:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2652 00000100:00080000:3.0:1769075663.443375:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075663.443381:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075663.443412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617166720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075663.443433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075663.443436:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075663.443437:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075663.443440:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2652 00000100:00080000:3.0:1769075663.443443:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075663.443445:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075663.443454:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075663.443467:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617166848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075663.443474:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617166720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075663.443481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075663.443487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075663.443489:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075663.443494:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2652 00000100:00080000:3.0:1769075663.443508:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075663.443513:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075663.443518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075663.443528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617166976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075663.443536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769075663.443541:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075663.443542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075663.443546:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075663.443551:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2652 00000100:00080000:3.0:1769075663.443555:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075663.443558:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075663.443562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075663.443571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617167104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075663.443577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075663.443580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075663.443582:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075663.443587:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2652 00000100:00100000:1.0:1769075663.443589:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075663/real 1769075663] req@ffff9b0352a19f80 x1855007617166720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075663.443590:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075663.443595:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075663.443601:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075663.443614:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617167232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075663.443623:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617166720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075663.443637:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075663.443643:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075663.443648:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617166720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075663.443680:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075663.443706:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075663.443719:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075663/real 1769075663] req@ffff9b0352a1a300 x1855007617166848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075663.443733:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617166848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075663.443743:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075663.443746:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075663.443749:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075663.443756:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075663.443774:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075663.443784:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075663/real 1769075663] req@ffff9b0352a18e00 x1855007617166976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075663.443796:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617166976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075663.443804:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075663.443807:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075663.443811:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617166976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075663.443818:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075663.443836:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075663.443847:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075663/real 1769075663] req@ffff9b0352a18700 x1855007617167104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075663.443859:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617167104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075663.443867:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075663.443870:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075663.443874:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075663.443879:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075663.443892:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075663.443901:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075663/real 1769075663] req@ffff9b0352a1aa00 x1855007617167232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075663.443913:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617167232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075663.443921:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075663.443924:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075663.443927:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075663.955342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075664.339350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075664.339354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075664.339423:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075664.979348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075665.363317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075665.363336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075665.363388:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075666.003355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075666.387314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075666.387346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075666.387386:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075667.027349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075667.411314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075667.411314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075667.411367:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075668.051349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075668.435324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075668.435344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075668.435442:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075668.563372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075668.563385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075668.563392:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075668.563403:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2657 00000100:00080000:1.0:1769075668.563415:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075668.563424:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075668.563477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617167360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075668.563510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075668.563513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075668.563515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075668.563521:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2657 00000100:00080000:1.0:1769075668.563525:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075668.563528:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075668.563539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075668.563556:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617167488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075668.563567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075668.563570:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075668.563572:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075668.563578:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2657 00000100:00080000:1.0:1769075668.563581:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075668.563585:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075668.563589:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075668.563600:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617167616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075668.563605:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617167360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075668.563621:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075668.563624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075668.563626:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075668.563631:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2657 00000100:00080000:1.0:1769075668.563634:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075668.563637:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075668.563641:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075668.563652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617167744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075668.563663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075668.563668:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075668.563670:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075668.563679:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2657 00000100:00080000:1.0:1769075668.563686:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075668.563690:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075668.563698:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075668.563710:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617167872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075668.563717:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075668.563883:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075668/real 1769075668] req@ffff9b0343dee300 x1855007617167360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075668.563953:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617167360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075668.564141:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075668.564145:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075668.564150:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617167360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075668.564158:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075668.564224:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075668.564269:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075668/real 1769075668] req@ffff9b0343def100 x1855007617167488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075668.564293:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617167488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075668.564304:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075668.564308:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075668.564311:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075668.564342:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075668.564357:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075668.564366:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075668/real 1769075668] req@ffff9b0343ded880 x1855007617167616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075668.564374:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617167616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075668.564380:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075668.564382:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075668.564385:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075668.564388:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075668.564399:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075668.564404:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075668/real 1769075668] req@ffff9b0343def800 x1855007617167744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075668.564412:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617167744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075668.564417:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075668.564419:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075668.564420:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075668.564424:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075668.564433:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075668.564438:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075668/real 1769075668] req@ffff9b0343deea00 x1855007617167872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075668.564445:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617167872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075668.564451:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075668.564454:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075668.564456:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617167872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075669.075368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075669.459307:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075669.459310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075669.459443:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075670.099371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075670.483336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075670.483344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075670.483439:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075671.123352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075671.507370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075671.507385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075671.507438:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075672.147348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075672.531314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075672.531320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075672.531383:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075673.171305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075673.555418:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075673.555418:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075673.555557:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075673.683434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075673.683448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075673.683454:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075673.683468:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2662 00000100:00080000:3.0:1769075673.683480:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075673.683489:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075673.683539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617168000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075673.683580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075673.683585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075673.683587:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075673.683593:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2662 00000100:00080000:3.0:1769075673.683597:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075673.683604:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075673.683617:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075673.683639:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617168128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075673.683642:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617168000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075673.683671:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075673.683676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075673.683679:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075673.683687:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2662 00000100:00080000:3.0:1769075673.683692:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075673.683696:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075673.683703:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075673.683718:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617168256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075673.683730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075673.683735:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075673.683737:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769075673.683748:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075673.683749:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2662 00000100:00080000:3.0:1769075673.683761:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075673.683768:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075673.683776:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075673.683791:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617168384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075673.683804:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075673.683810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769075673.683816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075673/real 1769075673] req@ffff9b0352a1aa00 x1855007617168000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075673.683817:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075673.683832:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2662 00000100:00080000:3.0:1769075673.683843:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075673.683854:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075673.683863:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075673.683871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617168000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075673.683882:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617168512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075673.683894:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075673.683910:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075673.683918:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617168000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075673.683950:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075673.684022:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075673.684044:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075673/real 1769075673] req@ffff9b0352a18700 x1855007617168128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075673.684061:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617168128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075673.684073:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075673.684077:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075673.684081:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075673.684090:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075673.684115:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075673.684126:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075673/real 1769075673] req@ffff9b0352a18e00 x1855007617168256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075673.684150:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617168256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075673.684159:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075673.684163:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075673.684167:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075673.684174:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075673.684193:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075673.684203:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075673/real 1769075673] req@ffff9b0352a1a300 x1855007617168384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075673.684217:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617168384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075673.684227:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075673.684230:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075673.684265:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075673.684273:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075673.684297:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075673.684317:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075673/real 1769075673] req@ffff9b0352a19f80 x1855007617168512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075673.684341:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617168512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075673.684369:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075673.684375:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075673.684382:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075674.195321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075674.579305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075674.579315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075674.579366:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075675.219329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075675.603404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075675.603408:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075675.603548:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075676.243334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075676.627311:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075676.627312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075676.627420:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075677.267328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075677.651332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075677.651350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075677.651373:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075678.291341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075678.675391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075678.675391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075678.675461:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075678.803375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075678.803388:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075678.803394:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075678.803406:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2667 00000100:00080000:2.0:1769075678.803417:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075678.803425:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075678.803469:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617168640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075678.803509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075678.803514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075678.803517:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075678.803522:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2667 00000100:00080000:2.0:1769075678.803527:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075678.803532:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075678.803543:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075678.803564:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617168768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075678.803580:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617168640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075678.803581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075678.803592:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075678.803595:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075678.803613:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2667 00000100:00080000:2.0:1769075678.803618:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075678.803622:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075678.803628:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075678.803640:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617168896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075678.803654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075678.803658:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075678.803662:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769075678.803664:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075678.803685:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2667 00000100:00080000:2.0:1769075678.803694:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075678.803700:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075678.803707:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075678.803733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617169024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075678.803745:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075678.803751:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769075678.803754:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075678/real 1769075678] req@ffff9b03482c5500 x1855007617168640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075678.803754:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075678.803764:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2667 00000100:00080000:2.0:1769075678.803773:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075678.803801:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075678.803814:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075678.803841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617168640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075678.803842:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617169152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075678.803909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075678.803917:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075678.803921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617168640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075678.803945:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075678.803967:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075678.803978:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075678/real 1769075678] req@ffff9b03482c4380 x1855007617168768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075678.803988:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617168768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075678.803995:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075678.803997:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075678.804000:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075678.804004:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075678.804015:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075678.804022:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075678/real 1769075678] req@ffff9b03482c4a80 x1855007617168896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075678.804030:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617168896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075678.804036:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075678.804038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075678.804040:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617168896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075678.804045:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075678.804056:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075678.804062:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075678/real 1769075678] req@ffff9b03482c6300 x1855007617169024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075678.804071:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617169024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075678.804076:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075678.804078:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075678.804081:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075678.804085:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075678.804095:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075678.804101:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075678/real 1769075678] req@ffff9b03482c6d80 x1855007617169152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075678.804109:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617169152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075678.804115:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075678.804117:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075678.804119:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075679.315301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075679.699373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075679.699385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075679.699424:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075680.339330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075680.723330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075680.723333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075680.723427:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075681.363320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075681.747309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075681.747313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075681.747361:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075682.387360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075682.771304:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075682.771327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075682.771375:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075683.411387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075683.795315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075683.795319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075683.795434:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075683.923376:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075683.923387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075683.923392:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075683.923402:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2673 00000100:00080000:1.0:1769075683.923412:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075683.923419:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075683.923457:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617169280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075683.923488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075683.923492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075683.923494:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075683.923499:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2673 00000100:00080000:1.0:1769075683.923503:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075683.923508:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075683.923517:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075683.923529:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617169408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075683.923541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075683.923546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075683.923548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075683.923553:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2673 00000100:00080000:1.0:1769075683.923557:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075683.923561:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075683.923567:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075683.923585:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617169536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075683.923585:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617169280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075683.923613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075683.923618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075683.923622:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075683.923628:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2673 00000100:00080000:1.0:1769075683.923632:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075683.923636:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075683.923643:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075683.923657:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617169664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075683.923667:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075683.923673:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075683.923675:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075683.923680:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2673 00000100:00080000:1.0:1769075683.923684:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769075683.923687:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075683.923694:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075683.923701:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075683.923720:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617169792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075683.923750:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075683/real 1769075683] req@ffff9b0343deed80 x1855007617169280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075683.923769:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617169280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075683.923779:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075683.923783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075683.923787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617169280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075683.923793:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075683.923828:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075683.923839:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075683/real 1769075683] req@ffff9b0343deca80 x1855007617169408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075683.923853:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617169408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075683.923863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075683.923867:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075683.923870:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075683.923903:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075683.923924:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075683.923933:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075683/real 1769075683] req@ffff9b0343dece00 x1855007617169536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075683.923942:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617169536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075683.923952:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075683.923955:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075683.923957:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075683.923961:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075683.923973:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075683.923979:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075683/real 1769075683] req@ffff9b0343defb80 x1855007617169664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075683.923987:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617169664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075683.923992:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075683.923994:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075683.923996:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075683.923999:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075683.924009:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075683.924014:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075683/real 1769075683] req@ffff9b0343dee680 x1855007617169792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075683.924022:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617169792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075683.924027:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075683.924028:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075683.924031:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617169792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075684.435337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075684.819321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075684.819321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075684.819381:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075685.459338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075685.843335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075685.843338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075685.843427:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075686.483317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075686.867300:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075686.867301:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075686.867332:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075687.507333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075687.891311:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075687.891335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075687.891369:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075688.531321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075688.915349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075688.915349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075688.915382:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075689.043327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075689.043337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075689.043342:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075689.043351:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2678 00000100:00080000:1.0:1769075689.043358:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075689.043364:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075689.043395:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617169920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075689.043425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075689.043427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075689.043429:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075689.043432:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2678 00000100:00080000:1.0:1769075689.043434:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075689.043437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075689.043444:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075689.043456:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617170048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075689.043464:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075689.043467:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075689.043468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075689.043471:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2678 00000100:00080000:1.0:1769075689.043473:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075689.043475:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075689.043480:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075689.043492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617170176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075689.043493:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617169920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075689.043521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075689.043523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075689.043524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075689.043527:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2678 00000100:00080000:1.0:1769075689.043529:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075689.043532:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075689.043537:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075689.043548:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617170304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075689.043554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075689.043556:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075689.043557:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075689.043560:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2678 00000100:00080000:1.0:1769075689.043562:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075689.043566:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075689.043569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075689.043577:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617170432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075689.043608:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075689.043656:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075689/real 1769075689] req@ffff9b0343def480 x1855007617169920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075689.043672:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617169920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075689.043682:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075689.043685:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075689.043689:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617169920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075689.043695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075689.043713:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075689.043747:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075689/real 1769075689] req@ffff9b0343ded500 x1855007617170048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075689.043757:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617170048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075689.043765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075689.043768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075689.043770:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075689.043796:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075689.043812:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075689.043820:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075689/real 1769075689] req@ffff9b0343dee680 x1855007617170176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075689.043827:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617170176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075689.043833:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075689.043836:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075689.043838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075689.043842:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075689.043852:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075689.043857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075689/real 1769075689] req@ffff9b0343defb80 x1855007617170304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075689.043865:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617170304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075689.043870:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075689.043872:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075689.043875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075689.043878:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075689.043888:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075689.043893:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075689/real 1769075689] req@ffff9b0343dece00 x1855007617170432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075689.043900:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617170432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075689.043905:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075689.043907:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075689.043909:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075689.555365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075689.939330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075689.939331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075689.939444:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075690.579329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075690.963366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075690.963370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075690.963424:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075691.603344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075691.987351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075691.987371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075691.987410:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075692.627339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075693.011354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075693.011354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075693.011412:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075693.651377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075694.035294:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075694.035309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075694.035401:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075694.163375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075694.163388:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075694.163393:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075694.163404:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2683 00000100:00080000:1.0:1769075694.163415:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075694.163423:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075694.163483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617170560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075694.163525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075694.163530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075694.163532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075694.163537:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2683 00000100:00080000:1.0:1769075694.163541:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075694.163545:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075694.163554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075694.163567:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617170688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075694.163578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075694.163581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075694.163583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075694.163587:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2683 00000100:00080000:1.0:1769075694.163590:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075694.163593:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075694.163597:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075694.163609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617170816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075694.163619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075694.163622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075694.163623:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075694.163630:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2683 00000100:00080000:1.0:1769075694.163675:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075694.163684:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075694.163692:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075694.163697:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617170560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075694.163729:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617170944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075694.163744:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075694.163749:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075694.163751:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075694.163758:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2683 00000100:00080000:1.0:1769075694.163763:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075694.163768:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075694.163775:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075694.163785:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075694.163807:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617171072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075694.163841:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075694/real 1769075694] req@ffff9b0343deca80 x1855007617170560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075694.163857:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617170560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075694.163865:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075694.163868:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075694.163871:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617170560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075694.163876:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075694.163891:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075694.163898:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075694/real 1769075694] req@ffff9b0343deed80 x1855007617170688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075694.163906:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617170688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075694.163912:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075694.163914:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075694.163916:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075694.163926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075694.163936:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075694.163942:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075694/real 1769075694] req@ffff9b0343deea00 x1855007617170816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075694.163957:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617170816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075694.163963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075694.163965:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075694.163967:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075694.163987:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075694.163999:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075694.164005:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075694/real 1769075694] req@ffff9b0343def800 x1855007617170944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075694.164013:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617170944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075694.164018:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075694.164019:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075694.164021:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617170944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075694.164025:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075694.164034:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075694.164039:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075694/real 1769075694] req@ffff9b0343ded880 x1855007617171072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075694.164046:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617171072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075694.164051:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075694.164052:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075694.164054:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075694.675385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075695.059386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075695.059399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075695.059698:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075695.699344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075696.083486:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769075696.083798:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075696.084004:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075696.723363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075697.107383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075697.107479:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075697.107559:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075697.747388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769075698.131894:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075698.131902:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075698.131902:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075698.771307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769075699.155342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075699.155440:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075699.155501:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:1.0:1769075699.283308:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075699.283317:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075699.283321:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075699.283329:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2688 00000100:00080000:1.0:1769075699.283337:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075699.283343:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075699.283381:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617171200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075699.283412:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075699.283414:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075699.283416:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075699.283419:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2688 00000100:00080000:1.0:1769075699.283421:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075699.283423:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075699.283431:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075699.283440:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617171328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075699.283448:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075699.283462:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075699.283465:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769075699.283468:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617171200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075699.283469:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2688 00000100:00080000:1.0:1769075699.283474:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075699.283478:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075699.283481:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075699.283489:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617171456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075699.283495:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075699.283498:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075699.283499:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075699.283503:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2688 00000100:00080000:1.0:1769075699.283505:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075699.283508:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075699.283511:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075699.283517:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617171584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075699.283524:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075699.283526:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075699.283527:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769075699.283531:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075699.283535:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2688 00000100:00080000:1.0:1769075699.283539:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075699.283542:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075699.283546:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075699.283580:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617171712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075699.283600:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075699/real 1769075699] req@ffff9b0343def100 x1855007617171200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075699.283615:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617171200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075699.283634:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075699.283638:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075699.283640:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617171200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075699.283659:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075699.283684:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075699.283693:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075699/real 1769075699] req@ffff9b0343dee300 x1855007617171328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075699.283701:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617171328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075699.283707:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075699.283710:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075699.283712:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075699.283716:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075699.283728:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075699.283735:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075699/real 1769075699] req@ffff9b0343dec380 x1855007617171456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075699.283742:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617171456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075699.283747:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075699.283749:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075699.283751:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075699.283755:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075699.283764:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075699.283770:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075699/real 1769075699] req@ffff9b0343ded180 x1855007617171584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075699.283776:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617171584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075699.283782:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075699.283783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075699.283785:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075699.283789:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075699.283798:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075699.283803:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075699/real 1769075699] req@ffff9b0343dedf80 x1855007617171712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075699.283815:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617171712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075699.283820:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075699.283821:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075699.283823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075699.795364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075700.179442:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075700.179499:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075700.179647:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075700.819308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075701.203305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075701.203315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075701.203404:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075701.843295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075702.227397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075702.227403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075702.227447:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075702.867457:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075703.251382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075703.251446:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075703.251478:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075703.891305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075704.275404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075704.275426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075704.275526:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075704.403532:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075704.403562:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075704.403569:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075704.403586:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2693 00000100:00080000:3.0:1769075704.403603:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075704.403614:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075704.403666:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617171840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075704.403713:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075704.403719:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075704.403721:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075704.403726:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2693 00000100:00080000:3.0:1769075704.403731:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075704.403736:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075704.403752:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075704.403802:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617171968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075704.403822:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075704.403829:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075704.403832:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075704.403841:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2693 00000100:00080000:3.0:1769075704.403861:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075704.403871:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075704.403884:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617171840:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769075704.403887:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075704.403985:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617172096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075704.404005:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075704.404013:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075704.404018:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075704.404020:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075704.404033:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2693 00000100:00080000:3.0:1769075704.404039:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075704.404045:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075704.404110:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075704/real 1769075704] req@ffff9b0352a18a80 x1855007617171840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075704.404157:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617171840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075704.404173:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075704.404184:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075704.404191:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617171840:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769075704.404212:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075704.404217:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075704.404351:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617172224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075704.404371:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075704.404375:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075704.404380:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075704.404386:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2693 00000100:00080000:3.0:1769075704.404392:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075704.404403:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769075704.404413:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769075704.404418:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075704.404444:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617172352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075704.404456:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075704/real 1769075704] req@ffff9b0352a1b480 x1855007617171968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075704.404538:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617171968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075704.404554:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075704.404560:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075704.404564:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617171968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075704.404609:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075704.404641:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075704.404657:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075704/real 1769075704] req@ffff9b0352a1ad80 x1855007617172096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075704.404671:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617172096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075704.404681:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075704.404685:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075704.404689:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075704.404696:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075704.404745:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075704.404777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075704/real 1769075704] req@ffff9b0352a1b100 x1855007617172224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075704.404803:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617172224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075704.404821:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075704.404838:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075704.404845:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075704.404858:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075704.404889:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075704.404904:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075704/real 1769075704] req@ffff9b0352a1a680 x1855007617172352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075704.404918:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617172352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075704.404930:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075704.404934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075704.404938:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075704.915398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075705.299374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075705.299374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075705.299540:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075705.939382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075706.323426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075706.323454:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075706.323609:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075706.963510:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075707.347388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075707.347398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075707.347555:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075707.987530:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075708.371367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075708.371369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075708.371527:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075709.011435:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075709.395366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075709.395366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075709.395518:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075709.523414:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075709.523444:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075709.523449:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075709.523462:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2698 00000100:00080000:3.0:1769075709.523474:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075709.523482:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075709.523525:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617172480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075709.523562:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075709.523567:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075709.523569:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075709.523574:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2698 00000100:00080000:3.0:1769075709.523578:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075709.523584:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075709.523596:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075709.523617:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617172608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075709.523619:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617172480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075709.523663:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075709.523667:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075709.523670:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075709.523676:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2698 00000100:00080000:3.0:1769075709.523681:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075709.523685:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075709.523692:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075709.523706:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617172736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075709.523732:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075709.523735:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075709.523737:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075709.523739:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2698 00000100:00080000:3.0:1769075709.523743:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075709.523746:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075709.523751:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075709.523764:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617172864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075709.523766:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075709.523778:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075709.523782:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075709.523784:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075709.523788:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2698 00000100:00080000:3.0:1769075709.523792:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075709.523795:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075709.523799:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075709.523809:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617172992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075709.523830:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075709/real 1769075709] req@ffff9b0352a1b800 x1855007617172480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075709.523857:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617172480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075709.523868:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075709.523872:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075709.523875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617172480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075709.523899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075709.523922:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075709.523933:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075709/real 1769075709] req@ffff9b0352a18000 x1855007617172608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075709.523943:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617172608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075709.523951:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075709.523954:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075709.523956:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075709.523962:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075709.523975:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075709.523983:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075709/real 1769075709] req@ffff9b0352a19500 x1855007617172736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075709.523993:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617172736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075709.523999:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075709.524001:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075709.524004:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075709.524008:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075709.524021:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075709.524027:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075709/real 1769075709] req@ffff9b0352a19c00 x1855007617172864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075709.524037:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617172864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075709.524044:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075709.524046:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075709.524048:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075709.524053:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075709.524063:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075709.524070:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075709/real 1769075709] req@ffff9b0352a18380 x1855007617172992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075709.524080:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617172992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075709.524087:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075709.524089:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075709.524091:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617172992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075710.035325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075710.419314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075710.419314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075710.419384:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075711.059389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075711.443480:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075711.443495:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075711.443561:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075712.083374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075712.467394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075712.467441:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075712.467538:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075713.107477:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075713.491325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075713.491332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075713.491409:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075714.131361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075714.515361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075714.515361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075714.515447:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075714.643356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075714.643365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075714.643370:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075714.643378:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2703 00000100:00080000:3.0:1769075714.643386:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075714.643391:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075714.643424:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617173120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075714.643452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075714.643455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075714.643456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075714.643459:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2703 00000100:00080000:3.0:1769075714.643462:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075714.643464:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075714.643472:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075714.643482:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617173248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075714.643489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075714.643493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075714.643494:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075714.643496:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2703 00000100:00080000:3.0:1769075714.643498:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075714.643500:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075714.643503:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075714.643509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617173376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075714.643518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075714.643520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075714.643521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075714.643523:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2703 00000100:00080000:3.0:1769075714.643526:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075714.643531:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075714.643536:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075714.643541:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617173120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075714.643547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617173504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075714.643565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075714.643567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075714.643568:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075714.643571:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2703 00000100:00080000:3.0:1769075714.643573:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075714.643576:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075714.643580:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075714.643590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617173632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075714.643699:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617173248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075714.643761:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617173376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075714.643792:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617173504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075714.643820:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617173632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075715.155369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075715.539336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075715.539351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075715.539409:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075716.179306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075716.563377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075716.563395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075716.563455:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075717.203361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075717.587331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075717.587338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075717.587416:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769075717.715518:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769075717.715570:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769075717.715589:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769075717.715633:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769075717.715640:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769075717.715646:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769075717.715653:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:3.0:1769075717.715696:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075714/real 1769075717] req@ffff9b0352a19500 x1855007617173376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075717.715731:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617173376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075717.715755:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075717.715762:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075717.715765:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617173376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075717.715773:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075714/real 1769075717] req@ffff9b0352a18000 x1855007617173504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075717.715789:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617173504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075717.715798:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075717.715802:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075717.715806:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617173504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075717.715814:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075714/real 1769075717] req@ffff9b0352a1b800 x1855007617173632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075717.715829:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617173632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075717.715838:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075717.715842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075717.715845:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617173632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075717.715852:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075714/real 1769075717] req@ffff9b0352a18380 x1855007617173120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075717.715866:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617173120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075717.715876:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075717.715880:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075717.715884:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617173120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075717.715892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075714/real 1769075717] req@ffff9b0352a19c00 x1855007617173248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075717.715906:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617173248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075717.715916:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075717.715920:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075717.715923:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617173248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075718.227353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075718.611355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075718.611402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075718.611481:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075719.251303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075719.635331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075719.635349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075719.635415:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075719.763376:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075719.763385:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075719.763389:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075719.763398:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2708 00000100:00080000:1.0:1769075719.763406:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075719.763412:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075719.763442:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617173760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075719.763468:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075719.763472:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075719.763474:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075719.763477:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2708 00000100:00080000:1.0:1769075719.763479:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075719.763482:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075719.763489:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075719.763497:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617173888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075719.763510:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617173760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075719.763513:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075719.763519:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075719.763520:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075719.763524:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2708 00000100:00080000:1.0:1769075719.763538:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075719.763541:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075719.763544:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075719.763552:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617174016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075719.763558:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075719.763561:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075719.763563:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075719.763565:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2708 00000100:00080000:1.0:1769075719.763568:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075719.763570:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075719.763573:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075719.763581:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617174144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075719.763587:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075719.763599:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075719.763602:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075719.763604:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075719.763608:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2708 00000100:00080000:1.0:1769075719.763611:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075719.763613:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075719.763616:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075719.763624:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617174272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075719.763632:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075719/real 1769075719] req@ffff9b0343dec700 x1855007617173760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075719.763650:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617173760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075719.763658:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075719.763661:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075719.763665:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617173760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075719.763684:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617173888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075719.763698:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075719.763709:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075719/real 1769075719] req@ffff9b0343dec000 x1855007617173888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075719.763722:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617173888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075719.763731:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075719.763734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075719.763738:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617173888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075719.763745:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075719.763765:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075719.763776:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075719/real 1769075719] req@ffff9b0343dedf80 x1855007617174016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075719.763790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617174016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075719.763799:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075719.763802:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075719.763806:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075719.763813:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075719.763835:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075719.763850:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075719/real 1769075719] req@ffff9b0343ded180 x1855007617174144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075719.763863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617174144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075719.763871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075719.763873:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075719.763876:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075719.763882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075719.763900:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075719.763911:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075719/real 1769075719] req@ffff9b0343dec380 x1855007617174272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075719.763925:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617174272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075719.763933:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075719.763936:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075719.763940:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075720.275339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075720.659365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075720.659370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075720.659486:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075721.299321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075721.683372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075721.683385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075721.683478:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075722.323342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075722.707334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075722.707373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075722.707491:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075723.347347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075723.731356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075723.731403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075723.731496:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075724.371340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075724.755379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075724.755426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075724.755467:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075724.883402:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075724.883413:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075724.883418:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075724.883429:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2714 00000100:00080000:3.0:1769075724.883440:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075724.883466:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075724.883507:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617174400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075724.883541:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075724.883545:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075724.883547:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075724.883552:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2714 00000100:00080000:3.0:1769075724.883555:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075724.883559:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075724.883569:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075724.883583:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617174528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075724.883595:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075724.883619:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1769075724.883628:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617174400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075724.883628:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075724.883650:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2714 00000100:00080000:3.0:1769075724.883660:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075724.883668:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075724.883681:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075724.883726:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617174656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075724.883743:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075724.883754:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075724.883760:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075724.883764:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075724.883772:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2714 00000100:00080000:3.0:1769075724.883795:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075724.883801:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075724.883809:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075724.883818:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075724/real 1769075724] req@ffff9b0352a19c00 x1855007617174400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075724.883829:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617174784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075724.883856:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075724.883860:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769075724.883865:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617174400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075724.883866:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075724.883878:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2714 00000100:00080000:3.0:1769075724.883881:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075724.883884:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075724.883889:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075724.883901:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617174912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075724.883913:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075724.883925:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075724.883935:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617174400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075724.883989:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075724.884023:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075724.884038:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075724/real 1769075724] req@ffff9b0352a18380 x1855007617174528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075724.884069:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617174528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075724.884079:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075724.884083:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075724.884086:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075724.884093:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075724.884114:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075724.884190:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075724/real 1769075724] req@ffff9b0352a1b800 x1855007617174656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075724.884206:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617174656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075724.884225:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075724.884229:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075724.884233:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075724.884240:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075724.884319:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075724.884334:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075724/real 1769075724] req@ffff9b0352a18000 x1855007617174784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075724.884349:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617174784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075724.884359:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075724.884363:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075724.884378:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075724.884387:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075724.884407:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075724.884419:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075724/real 1769075724] req@ffff9b0352a19500 x1855007617174912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075724.884434:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617174912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075724.884443:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075724.884463:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075724.884468:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617174912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075725.395401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075725.779304:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075725.779397:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075725.779426:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075726.419423:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075726.803314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075726.803319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075726.803393:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075727.443373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075727.827379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075727.827430:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075727.827511:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075728.467397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075728.851350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075728.851398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075728.851471:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075729.491357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075729.875413:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075729.875416:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075729.875622:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075730.003337:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075730.003349:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075730.003354:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075730.003364:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2719 00000100:00080000:3.0:1769075730.003374:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075730.003380:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075730.003417:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617175040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075730.003481:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075730.003485:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075730.003487:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075730.003492:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2719 00000100:00080000:3.0:1769075730.003496:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075730.003500:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075730.003509:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075730.003522:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617175168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075730.003534:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075730.003539:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075730.003542:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075730.003546:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2719 00000100:00080000:3.0:1769075730.003550:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075730.003557:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617175040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075730.003558:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075730.003564:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075730.003579:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617175296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075730.003588:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075730.003592:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075730.003594:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075730.003597:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2719 00000100:00080000:3.0:1769075730.003601:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075730.003604:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075730.003609:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075730.003619:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617175424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075730.003627:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075730.003631:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075730.003633:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075730.003637:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2719 00000100:00080000:3.0:1769075730.003640:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075730.003643:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075730.003665:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075730.003676:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617175552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075730.003681:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075730.003734:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff9b0352a1a680 x1855007617175040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075730.003771:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617175040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075730.003782:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075730.003786:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075730.003791:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617175040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075730.003799:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075730.003823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075730.003865:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff9b0352a1b100 x1855007617175168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075730.003877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617175168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075730.003885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075730.003888:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075730.003891:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075730.003932:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075730.003952:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075730.003962:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff9b0352a1ad80 x1855007617175296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075730.003974:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617175296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075730.003982:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075730.003984:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075730.003988:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075730.003993:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075730.004007:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075730.004015:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff9b0352a1b480 x1855007617175424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075730.004025:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617175424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075730.004057:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075730.004062:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075730.004067:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075730.004092:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075730.004118:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075730.004134:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075730/real 1769075730] req@ffff9b0352a18a80 x1855007617175552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075730.004151:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617175552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075730.004162:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075730.004165:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075730.004168:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075730.515427:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075730.899387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075730.899403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075730.899470:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075731.539383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075731.923341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075731.923359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075731.923470:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075732.563403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075732.947339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075732.947339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075732.947413:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075733.587322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075733.971329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075733.971330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075733.971396:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075734.611378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075734.995328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075734.995328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075734.995392:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075735.123431:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075735.123443:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075735.123447:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075735.123458:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2724 00000100:00080000:1.0:1769075735.123468:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075735.123477:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075735.123513:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617175680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075735.123551:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075735.123554:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075735.123556:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075735.123559:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2724 00000100:00080000:1.0:1769075735.123562:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075735.123566:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075735.123574:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075735.123583:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617175808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075735.123591:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075735.123593:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075735.123595:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075735.123597:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2724 00000100:00080000:1.0:1769075735.123599:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075735.123601:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075735.123605:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075735.123612:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617175936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075735.123618:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075735.123621:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075735.123622:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075735.123625:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2724 00000100:00080000:1.0:1769075735.123629:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075735.123634:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075735.123636:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617175680:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769075735.123639:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075735.123661:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617176064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075735.123668:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075735.123670:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075735.123672:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075735.123674:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2724 00000100:00080000:1.0:1769075735.123677:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075735.123679:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075735.123683:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075735.123690:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617176192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075735.123721:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075735.123774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075735/real 1769075735] req@ffff9b0343dee300 x1855007617175680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075735.123789:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617175680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075735.123797:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075735.123800:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075735.123804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617175680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075735.123809:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075735.123827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075735.123835:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075735/real 1769075735] req@ffff9b0343def100 x1855007617175808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075735.123844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617175808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075735.123867:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075735.123871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075735.123875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075735.123882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075735.123899:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075735.123908:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075735/real 1769075735] req@ffff9b0343ded880 x1855007617175936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075735.123919:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617175936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075735.123927:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075735.123930:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075735.123932:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617175936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075735.124014:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075735.124031:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075735.124039:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075735/real 1769075735] req@ffff9b0343def800 x1855007617176064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075735.124047:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617176064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075735.124053:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075735.124055:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075735.124057:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075735.124079:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075735.124094:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075735.124103:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075735/real 1769075735] req@ffff9b0343deea00 x1855007617176192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075735.124115:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617176192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075735.124123:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075735.124126:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075735.124129:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075735.635358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075736.019399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075736.019415:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075736.019525:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075736.659387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075737.043359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075737.043410:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075737.043492:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075737.683384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075738.067355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075738.067363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075738.067435:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075738.707343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075739.091364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075739.091393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075739.091536:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075739.731426:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075740.115387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075740.115391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075740.115543:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075740.243428:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075740.243440:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075740.243447:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075740.243461:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2729 00000100:00080000:1.0:1769075740.243476:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075740.243496:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075740.243543:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617176320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075740.243589:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075740.243594:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075740.243597:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075740.243603:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2729 00000100:00080000:1.0:1769075740.243608:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075740.243614:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075740.243628:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075740.243652:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617176320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075740.243659:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617176448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075740.243689:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075740.243697:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075740.243701:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075740.243711:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2729 00000100:00080000:1.0:1769075740.243721:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075740.243731:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075740.243743:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075740.243765:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075740.243775:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617176576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075740.243792:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075740.243797:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075740.243800:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075740.243806:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2729 00000100:00080000:1.0:1769075740.243813:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075740.243820:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075740.243829:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075740.243843:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075740/real 1769075740] req@ffff9b0343deed80 x1855007617176320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075740.243851:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617176704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075740.243915:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075740.243919:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075740.243921:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075740.243927:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2729 00000100:00080000:1.0:1769075740.243931:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075740.243936:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075740.243942:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075740.243956:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617176832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075740.244227:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617176320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075740.244365:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075740.244378:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075740.244390:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617176320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075740.244436:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075740.244490:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075740.244518:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075740/real 1769075740] req@ffff9b0343deca80 x1855007617176448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075740.244547:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617176448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075740.244560:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075740.244564:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075740.244569:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075740.244578:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075740.244607:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075740.244621:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075740/real 1769075740] req@ffff9b0343dece00 x1855007617176576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075740.244636:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617176576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075740.244646:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075740.244650:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075740.244654:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075740.244661:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075740.244680:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075740.244691:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075740/real 1769075740] req@ffff9b0343defb80 x1855007617176704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075740.244705:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617176704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075740.244724:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075740.244728:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075740.244732:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075740.244738:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075740.244756:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075740.244767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075740/real 1769075740] req@ffff9b0343dee680 x1855007617176832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075740.244780:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617176832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075740.244790:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075740.244793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075740.244797:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617176832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075740.755327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075741.139321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075741.139327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075741.139383:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075741.779330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075742.163352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075742.163425:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075742.163426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075742.803331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075743.187362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075743.187413:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075743.187419:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075743.827426:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075744.211343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075744.211371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075744.211479:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075744.851397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075745.235400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075745.235401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075745.235466:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075745.363370:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075745.363385:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075745.363392:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075745.363422:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2734 00000100:00080000:1.0:1769075745.363442:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075745.363454:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075745.363520:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617176960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075745.363572:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075745.363580:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075745.363583:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075745.363593:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2734 00000100:00080000:1.0:1769075745.363607:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075745.363618:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075745.363638:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075745.363653:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617176960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075745.363678:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617177088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075745.363701:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075745.363706:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075745.363711:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075745.363720:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2734 00000100:00080000:1.0:1769075745.363730:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075745.363740:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075745.363754:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075745.363761:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075745.363830:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617177216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075745.363850:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075745/real 1769075745] req@ffff9b0343ded500 x1855007617176960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075745.363870:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075745.363898:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075745.363911:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075745.363936:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2734 00000100:00080000:1.0:1769075745.363946:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075745.363955:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617176960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075745.363963:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075745.363990:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1769075745.363994:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075745.364007:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075745.364024:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617177344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075745.364037:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617176960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075745.364057:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075745.364064:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075745.364068:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075745.364075:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2734 00000100:00080000:1.0:1769075745.364082:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075745.364094:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075745.364096:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177088:192.168.204.160@tcp:38:kworker.0 00000008:00100000:1.0:1769075745.364130:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075745.364157:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075745.364174:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617177472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075745.364182:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075745/real 1769075745] req@ffff9b0343def480 x1855007617177088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075745.364286:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617177088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075745.364321:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075745.364328:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075745.364335:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075745.364353:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075745.364381:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075745.364395:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075745/real 1769075745] req@ffff9b0343dee680 x1855007617177216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075745.364410:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617177216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075745.364428:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075745.364433:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075745.364437:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075745.364444:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075745.364462:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075745.364473:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075745/real 1769075745] req@ffff9b0343defb80 x1855007617177344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075745.364487:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617177344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075745.364496:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075745.364499:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075745.364503:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075745.364523:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075745.364541:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075745.364553:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075745/real 1769075745] req@ffff9b0343dece00 x1855007617177472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075745.364566:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617177472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075745.364575:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075745.364578:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075745.364583:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075745.875402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075746.259387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075746.259391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075746.259469:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075746.899374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075747.283374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075747.283547:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075747.283605:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075747.923373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075748.307366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075748.307366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075748.307519:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075748.947403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075749.331318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075749.331322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075749.331410:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075749.971345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075750.355346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075750.355386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075750.355513:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075750.483355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075750.483364:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075750.483368:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075750.483376:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2739 00000100:00080000:0.0:1769075750.483385:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075750.483391:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075750.483427:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617177600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075750.483473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075750.483478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075750.483480:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075750.483486:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2739 00000100:00080000:0.0:1769075750.483490:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075750.483495:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075750.483506:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075750.483521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617177728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075750.483544:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075750.483548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075750.483560:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075750.483565:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2739 00000100:00080000:0.0:1769075750.483571:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075750.483576:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075750.483582:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075750.483594:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617177856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075750.483610:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617177600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075750.483613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075750.483628:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075750.483631:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075750.483637:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2739 00000100:00080000:0.0:1769075750.483642:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075750.483646:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075750.483653:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075750.483669:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617177984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075750.483681:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075750.483685:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075750.483687:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075750.483776:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2739 00000800:00000100:3.0:1769075750.483784:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075750.483785:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075750.483794:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075750.483827:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075750.483854:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617178112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075750.483899:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075750/real 1769075750] req@ffff9b0365a38a80 x1855007617177600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075750.483919:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617177600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075750.483928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075750.483932:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075750.483935:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617177600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075750.483942:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075750.483969:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075750.483979:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075750/real 1769075750] req@ffff9b0365a39c00 x1855007617177728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075750.483988:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617177728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075750.483996:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075750.483998:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075750.484001:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075750.484020:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075750.484041:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075750.484048:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075750/real 1769075750] req@ffff9b0365a38e00 x1855007617177856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075750.484057:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617177856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075750.484063:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075750.484065:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075750.484067:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075750.484072:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075750.484092:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075750.484104:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075750/real 1769075750] req@ffff9b0365a3b480 x1855007617177984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075750.484120:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617177984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075750.484130:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075750.484134:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075750.484139:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617177984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075750.484146:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617178112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075750.484166:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075750.484175:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075750/real 1769075750] req@ffff9b0365a3aa00 x1855007617178112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075750.484184:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617178112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075750.484210:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075750.484213:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075750.484217:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617178112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075750.995310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075751.379332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075751.379352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075751.379441:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075752.019346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075752.403320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075752.403322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075752.403362:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075753.043343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075753.427363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075753.427369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075753.427414:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075754.067330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075754.451303:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075754.451304:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075754.451385:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075755.091331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075755.475336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075755.475336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075755.475396:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075755.603309:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075755.603317:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075755.603322:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075755.603330:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2744 00000100:00080000:3.0:1769075755.603338:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075755.603344:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075755.603377:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617178240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075755.603409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075755.603414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075755.603416:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075755.603421:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2744 00000100:00080000:3.0:1769075755.603426:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075755.603432:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075755.603441:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617178240:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769075755.603442:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075755.603464:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617178368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075755.603476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075755.603481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075755.603483:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075755.603486:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2744 00000100:00080000:3.0:1769075755.603492:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075755.603496:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075755.603500:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769075755.603502:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075755.603524:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617178496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075755.603533:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075755.603536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075755.603537:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075755.603541:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2744 00000100:00080000:3.0:1769075755.603545:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075755.603548:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075755.603552:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075755/real 1769075755] req@ffff9b0352a19f80 x1855007617178240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769075755.603554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075755.603568:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617178624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075755.603574:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617178240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075755.603591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075755.603598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075755.603600:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075755.603600:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075755.603623:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2744 00000100:00080000:3.0:1769075755.603630:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075755.603631:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769075755.603636:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075755.603638:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617178240:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769075755.603643:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075755.603673:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617178752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075755.603685:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617178368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075755.603724:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075755.603745:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075755/real 1769075755] req@ffff9b0352a1a300 x1855007617178368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075755.603771:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617178368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075755.603777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075755.603779:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075755.603782:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617178368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075755.603786:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617178496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075755.603797:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075755.603803:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075755/real 1769075755] req@ffff9b0352a18e00 x1855007617178496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075755.603811:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617178496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075755.603836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075755.603839:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075755.603843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617178496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075755.603850:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617178624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075755.603869:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075755.603894:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075755/real 1769075755] req@ffff9b0352a18700 x1855007617178624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075755.603902:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617178624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075755.603907:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075755.603909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075755.603911:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617178624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075755.603921:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617178752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075755.603931:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075755.603938:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075755/real 1769075755] req@ffff9b0352a1aa00 x1855007617178752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075755.603945:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617178752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075755.603950:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075755.603951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075755.603953:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617178752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075756.115321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075756.499310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075756.499326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075756.499418:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075757.139359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075757.523332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075757.523336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075757.523438:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075758.163331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075758.547342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075758.547342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075758.547417:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075759.187292:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075759.571385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075759.571432:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075759.571564:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075760.211326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075760.595325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075760.595327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075760.595414:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075760.723330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075760.723340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075760.723345:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075760.723356:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2749 00000100:00080000:3.0:1769075760.723376:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075760.723384:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075760.723422:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617178880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075760.723453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075760.723462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075760.723464:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075760.723469:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2749 00000100:00080000:3.0:1769075760.723473:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075760.723478:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075760.723488:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075760.723500:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617178880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075760.723504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617179008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075760.723532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075760.723537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075760.723539:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075760.723544:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2749 00000100:00080000:3.0:1769075760.723548:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075760.723553:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075760.723570:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075760.723582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617179136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075760.723593:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075760.723596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075760.723601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075760.723604:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075760.723609:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2749 00000100:00080000:3.0:1769075760.723614:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075760.723618:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075760.723624:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075760.723636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617179264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075760.723643:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075760/real 1769075760] req@ffff9b0352a1aa00 x1855007617178880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075760.723653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075760.723659:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075760.723661:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075760.723665:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617178880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075760.723669:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2749 00000100:00080000:0.0:1769075760.723679:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075760.723684:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075760.723689:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617178880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075760.723693:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075760.723701:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075760.723707:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075760.723711:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075760.723724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617179392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075760.723733:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075760.723747:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075760/real 1769075760] req@ffff9b0352a18700 x1855007617179008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075760.723763:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617179008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075760.723777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075760.723779:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075760.723781:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075760.723785:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075760.723796:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075760.723802:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075760/real 1769075760] req@ffff9b0352a18e00 x1855007617179136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075760.723809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617179136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075760.723814:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075760.723816:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075760.723818:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075760.723821:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075760.723831:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075760.723837:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075760/real 1769075760] req@ffff9b0352a1a300 x1855007617179264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075760.723844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617179264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075760.723849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075760.723850:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075760.723853:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075760.723863:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075760.723872:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075760.723878:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075760/real 1769075760] req@ffff9b0352a19f80 x1855007617179392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075760.723885:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617179392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075760.723890:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075760.723892:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075760.723894:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075761.235449:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075761.619453:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075761.619533:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075761.619545:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075762.259323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075762.643376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769075762.643440:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075762.643456:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075763.283304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075763.667324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769075763.667437:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075763.667547:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075764.307311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075764.691436:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075764.691578:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075764.691618:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075765.331312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075765.715338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075765.715362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075765.715473:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075765.843317:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075765.843327:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075765.843331:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075765.843339:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2755 00000100:00080000:3.0:1769075765.843350:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075765.843357:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075765.843389:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617179520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075765.843412:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075765.843415:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075765.843416:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075765.843419:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2755 00000100:00080000:3.0:1769075765.843422:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075765.843424:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075765.843443:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075765.843455:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617179648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075765.843463:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075765.843466:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075765.843468:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075765.843471:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2755 00000100:00080000:3.0:1769075765.843474:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075765.843476:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075765.843482:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075765.843486:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617179520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075765.843493:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617179776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075765.843503:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075765.843505:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075765.843506:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075765.843508:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2755 00000100:00080000:3.0:1769075765.843510:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075765.843513:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075765.843518:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075765.843526:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617179904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075765.843532:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075765.843536:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075765.843537:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075765.843539:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2755 00000100:00080000:3.0:1769075765.843542:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075765.843544:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075765.843548:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075765.843556:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617180032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075765.843577:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075765.843628:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075765/real 1769075765] req@ffff9b0352a18a80 x1855007617179520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075765.843644:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617179520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075765.843654:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075765.843658:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075765.843661:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617179520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075765.843670:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075765.843695:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075765.843708:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075765/real 1769075765] req@ffff9b0352a1b480 x1855007617179648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075765.843723:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617179648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075765.843740:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075765.843743:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075765.843745:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075765.843773:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075765.843794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075765.843806:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075765/real 1769075765] req@ffff9b0352a1ad80 x1855007617179776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075765.843820:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617179776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075765.843829:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075765.843833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075765.843836:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075765.843844:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075765.843864:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075765.843875:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075765/real 1769075765] req@ffff9b0352a1b100 x1855007617179904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075765.843896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617179904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075765.843906:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075765.843909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075765.843913:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617179904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075765.843920:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075765.843938:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075765.843946:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075765/real 1769075765] req@ffff9b0352a1a680 x1855007617180032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075765.843954:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617180032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075765.843960:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075765.843962:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075765.843964:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075766.355297:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075766.739328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075766.739328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075766.739393:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075767.379389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075767.763362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075767.763469:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075767.763472:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075768.403324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075768.787376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075768.787376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075768.787468:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075769.427355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075769.811325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075769.811330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075769.811416:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075770.451303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075770.835332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075770.835332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075770.835396:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075770.963343:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075770.963363:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075770.963366:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075770.963374:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2760 00000100:00080000:1.0:1769075770.963383:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075770.963389:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075770.963420:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617180160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075770.963443:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075770.963445:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075770.963446:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075770.963450:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2760 00000100:00080000:1.0:1769075770.963452:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075770.963456:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075770.963464:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075770.963476:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617180160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075770.963477:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617180288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075770.963491:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075770.963494:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075770.963496:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075770.963499:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2760 00000100:00080000:1.0:1769075770.963502:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075770.963504:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075770.963507:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075770.963515:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617180416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075770.963521:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075770.963523:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075770.963524:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075770.963527:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2760 00000100:00080000:1.0:1769075770.963530:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075770.963533:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075770.963536:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769075770.963540:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075770.963546:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617180544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075770.963554:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075770.963557:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075770.963558:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075770.963561:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2760 00000100:00080000:1.0:1769075770.963564:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075770.963567:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075770.963571:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075770.963575:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075770/real 1769075770] req@ffff9b0343deca80 x1855007617180160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075770.963580:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617180672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075770.963606:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617180160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075770.963620:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075770.963623:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075770.963626:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617180160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075770.963644:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075770.963659:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075770.963666:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075770/real 1769075770] req@ffff9b0343deed80 x1855007617180288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075770.963675:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617180288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075770.963680:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075770.963683:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075770.963685:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075770.963688:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075770.963698:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075770.963703:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075770/real 1769075770] req@ffff9b0343deea00 x1855007617180416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075770.963710:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617180416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075770.963715:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075770.963717:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075770.963719:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075770.963723:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075770.963732:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075770.963737:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075770/real 1769075770] req@ffff9b0343def800 x1855007617180544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075770.963745:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617180544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075770.963750:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075770.963752:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075770.963753:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075770.963757:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075770.963765:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075770.963770:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075770/real 1769075770] req@ffff9b0343ded880 x1855007617180672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075770.963776:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617180672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075770.963781:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075770.963783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075770.963785:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075771.475400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075771.859308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075771.859309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075771.859389:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075772.499318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075772.883337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075772.883338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075772.883406:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075773.523353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075773.907343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075773.907343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075773.907456:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075774.547358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075774.931339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075774.931369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075774.931433:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075775.571338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075775.955344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075775.955372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075775.955423:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075776.083346:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075776.083357:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075776.083369:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075776.083380:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2765 00000100:00080000:1.0:1769075776.083392:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075776.083403:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075776.083446:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617180800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075776.083475:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075776.083478:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075776.083480:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075776.083483:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2765 00000100:00080000:1.0:1769075776.083487:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075776.083491:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075776.083503:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075776.083522:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617180800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075776.083527:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617180928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075776.083551:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075776.083556:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075776.083560:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075776.083570:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2765 00000100:00080000:1.0:1769075776.083577:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075776.083584:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075776.083593:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075776.083599:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075776.083621:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617181056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075776.083637:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075776.083641:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075776/real 1769075776] req@ffff9b0343def100 x1855007617180800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075776.083642:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075776.083649:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075776.083663:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2765 00000100:00080000:1.0:1769075776.083667:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075776.083670:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075776.083675:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075776.083684:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617181184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075776.083695:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075776.083697:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075776.083698:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075776.083700:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2765 00000100:00080000:1.0:1769075776.083702:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075776.083704:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075776.083707:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075776.083712:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617181312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075776.083751:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617180800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075776.083769:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075776.083775:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075776.083880:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617180800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075776.083906:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075776.083966:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075776.083992:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075776/real 1769075776] req@ffff9b0343dee300 x1855007617180928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075776.084011:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617180928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075776.084020:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075776.084023:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075776.084026:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617180928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075776.084032:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075776.084044:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075776.084051:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075776/real 1769075776] req@ffff9b0343dec380 x1855007617181056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075776.084058:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617181056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075776.084063:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075776.084065:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075776.084067:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075776.084071:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075776.084081:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075776.084087:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075776/real 1769075776] req@ffff9b0343ded180 x1855007617181184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075776.084094:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617181184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075776.084099:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075776.084101:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075776.084103:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075776.084106:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075776.084116:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075776.084121:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075776/real 1769075776] req@ffff9b0343dedf80 x1855007617181312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075776.084128:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617181312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075776.084133:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075776.084135:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075776.084136:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075776.595386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075776.979396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075776.979427:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075776.979506:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075777.619402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075778.003316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075778.003321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075778.003430:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075778.643351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075779.027372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075779.027374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075779.027513:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075779.667310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075780.051305:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075780.051334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075780.051410:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075780.691403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075781.075368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075781.075368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075781.075499:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075781.203488:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075781.203506:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075781.203514:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075781.203532:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2770 00000100:00080000:3.0:1769075781.203548:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075781.203560:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075781.205529:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617181440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075781.205562:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075781.205566:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075781.205569:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075781.205574:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2770 00000100:00080000:3.0:1769075781.205583:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075781.205590:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075781.205600:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075781.205612:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617181568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075781.205641:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075781.205645:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075781.205650:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075781.205657:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617181440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075781.205659:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2770 00000100:00080000:3.0:1769075781.205669:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075781.205681:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075781.205709:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075781.205727:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617181696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075781.205739:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075781.205744:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075781.205746:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075781.205769:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2770 00000100:00080000:3.0:1769075781.205780:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075781.205788:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075781.205798:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075781.205837:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617181824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075781.205848:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075781.205851:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075781.205852:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075781.205855:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2770 00000100:00080000:3.0:1769075781.205860:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075781.205863:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075781.205866:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075781.205875:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617181952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075781.205945:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075781.206000:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075781.206027:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075781.206057:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075781.715349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075782.099321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075782.099325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075782.099402:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075782.739306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075783.123373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075783.123373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075783.123493:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075783.763308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075784.147320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075784.147321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075784.147395:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075784.275678:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769075784.275746:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075784.275786:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075784.275858:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075784.275872:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075784.275904:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075784.275926:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769075784.275966:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075781/real 1769075784] req@ffff9b0352a18000 x1855007617181568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075784.276004:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617181568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075784.276021:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075784.276028:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075784.276034:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075784.276046:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075781/real 1769075784] req@ffff9b0352a1b800 x1855007617181696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075784.276067:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617181696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075784.276080:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075784.276086:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075784.276091:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075784.276100:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075781/real 1769075784] req@ffff9b0352a18380 x1855007617181824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075784.276117:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617181824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075784.276169:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075784.276176:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075784.276182:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075784.276193:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075781/real 1769075784] req@ffff9b0352a19c00 x1855007617181952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075784.276208:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617181952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075784.276216:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075784.276220:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075784.276223:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617181952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075784.276229:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075781/real 1769075784] req@ffff9b0352a19500 x1855007617181440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075784.276277:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617181440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075784.276289:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075784.276295:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075784.276316:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617181440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075784.787413:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075785.171322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075785.171376:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075785.171398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075785.811621:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075786.195301:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075786.195302:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075786.195403:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075786.323372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075786.323382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075786.323387:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075786.323397:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2775 00000100:00080000:2.0:1769075786.323406:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075786.323471:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075786.323525:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617182080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075786.323548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075786.323552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075786.323553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075786.323556:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2775 00000100:00080000:2.0:1769075786.323560:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075786.323563:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075786.323591:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075786.323613:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617182208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075786.323634:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617182080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075786.323635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075786.323648:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075786.323654:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075786.323663:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2775 00000100:00080000:2.0:1769075786.323687:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075786.323692:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075786.323698:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075786.323713:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617182336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075786.323724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075786.323728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075786.323731:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769075786.323739:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075786.323739:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2775 00000100:00080000:2.0:1769075786.323749:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075786.323755:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075786.323762:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075786.323779:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617182464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075786.323791:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075786/real 1769075786] req@ffff9b03482c5180 x1855007617182080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075786.323791:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075786.323802:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075786.323807:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075786.323822:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2775 00000100:00080000:2.0:1769075786.323834:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769075786.323840:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617182080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075786.323842:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075786.323852:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1769075786.323860:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769075786.323870:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617182592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075786.323871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075786.323881:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617182080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075786.323916:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075786.323968:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075786.323992:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075786/real 1769075786] req@ffff9b03482c4700 x1855007617182208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075786.324007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617182208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075786.324017:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075786.324022:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075786.324026:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075786.324034:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075786.324059:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075786.324072:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075786/real 1769075786] req@ffff9b03482c7100 x1855007617182336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075786.324086:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617182336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075786.324120:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075786.324125:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075786.324129:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075786.324136:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075786.324161:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075786.324175:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075786/real 1769075786] req@ffff9b03482c4000 x1855007617182464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075786.324189:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617182464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075786.324199:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075786.324203:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075786.324207:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075786.324214:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075786.324266:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075786.324280:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075786/real 1769075786] req@ffff9b03482c6a00 x1855007617182592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075786.324294:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617182592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075786.324303:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075786.324307:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075786.324312:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075786.835304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075787.219298:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075787.219333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075787.219452:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075787.859312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075788.243333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075788.243333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075788.243372:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075788.883312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075789.267302:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075789.267307:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075789.267395:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075789.907306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075790.291427:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075790.291437:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075790.291481:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075790.931338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075791.315359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075791.315365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075791.315486:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075791.443366:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075791.443380:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075791.443386:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075791.443398:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2780 00000100:00080000:3.0:1769075791.443410:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075791.443418:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075791.443503:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617182720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075791.443529:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075791.443533:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075791.443535:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075791.443538:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2780 00000100:00080000:3.0:1769075791.443543:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075791.443549:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075791.443558:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075791.443574:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617182848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075791.443575:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617182720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075791.443594:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075791.443597:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075791.443599:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075791.443602:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2780 00000100:00080000:3.0:1769075791.443605:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075791.443626:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075791.443632:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075791.443644:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617182976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075791.443655:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075791.443658:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075791.443660:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075791.443664:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2780 00000100:00080000:3.0:1769075791.443668:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075791.443671:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075791.443676:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075791.443686:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617183104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075791.443694:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075791.443698:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075791.443701:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075791.443705:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2780 00000800:00000100:0.0:1769075791.443735:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075791.443740:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075791.443746:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075791.443751:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075791.443762:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617183232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075791.443808:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075791/real 1769075791] req@ffff9b0352a19500 x1855007617182720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075791.443836:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617182720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075791.443851:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075791.443857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075791.443864:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617182720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075791.443889:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075791.443933:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075791.443950:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075791/real 1769075791] req@ffff9b0352a19c00 x1855007617182848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075791.443965:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617182848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075791.443974:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075791.443978:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075791.443982:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075791.444027:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075791.444099:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075791.444116:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075791/real 1769075791] req@ffff9b0352a18380 x1855007617182976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075791.444127:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617182976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075791.444135:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075791.444137:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075791.444140:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617182976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075791.444146:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075791.444164:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075791.444173:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075791/real 1769075791] req@ffff9b0352a1b800 x1855007617183104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075791.444182:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617183104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075791.444189:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075791.444191:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075791.444193:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075791.444198:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075791.444210:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075791.444216:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075791/real 1769075791] req@ffff9b0352a18000 x1855007617183232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075791.444226:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617183232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075791.444257:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075791.444260:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075791.444263:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075791.955374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075792.339434:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075792.339435:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075792.339465:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075792.979438:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075793.363384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075793.363398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075793.363538:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075794.003392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075794.387428:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075794.387430:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075794.387528:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075795.027309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075795.411314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075795.411321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075795.411384:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075796.051383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075796.435386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075796.435399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075796.435467:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075796.563465:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075796.563479:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075796.563484:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075796.563495:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2785 00000100:00080000:2.0:1769075796.563508:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075796.563517:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075796.563563:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617183360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075796.563603:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075796.563609:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075796.563611:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075796.563617:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2785 00000100:00080000:2.0:1769075796.563621:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075796.563626:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075796.563639:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075796.563654:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617183488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075796.563672:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617183360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075796.563673:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075796.563687:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075796.563692:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075796.563698:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2785 00000100:00080000:2.0:1769075796.563702:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075796.563707:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075796.563714:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075796.563733:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617183616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075796.563745:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075796.563749:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075796.563751:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075796.563755:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2785 00000100:00080000:2.0:1769075796.563761:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075796.563765:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075796.563772:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769075796.563777:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075796.563793:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617183744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075796.563808:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075796.563811:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075796.563814:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075796.563819:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2785 00000100:00080000:2.0:1769075796.563824:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075796.563829:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075796.563836:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075796.563840:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075796/real 1769075796] req@ffff9b03482c5f80 x1855007617183360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075796.563856:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617183872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075796.563876:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617183360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075796.563893:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075796.563900:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075796.563906:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617183360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075796.563973:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075796.564003:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075796.564019:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075796/real 1769075796] req@ffff9b03482c7b80 x1855007617183488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075796.564035:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617183488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075796.564045:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075796.564050:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075796.564054:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075796.564062:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075796.564081:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075796.564092:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075796/real 1769075796] req@ffff9b03482c4e00 x1855007617183616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075796.564106:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617183616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075796.564115:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075796.564118:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075796.564122:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075796.564129:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075796.564148:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075796.564158:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075796/real 1769075796] req@ffff9b03482c5c00 x1855007617183744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075796.564171:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617183744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075796.564180:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075796.564183:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075796.564187:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075796.564194:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075796.564211:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075796.564221:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075796/real 1769075796] req@ffff9b03482c6680 x1855007617183872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075796.564269:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617183872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075796.564280:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075796.564284:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075796.564288:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617183872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075797.075392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075797.459368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075797.459374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075797.459487:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075798.099401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075798.483403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075798.483428:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075798.483537:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075799.123410:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075799.507350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075799.507352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075799.507458:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075800.147400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075800.531370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075800.531377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075800.531544:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075801.171321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075801.555366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075801.555366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075801.555441:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075801.683382:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075801.683394:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075801.683399:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075801.683421:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2790 00000100:00080000:2.0:1769075801.683432:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075801.683441:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075801.683478:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617184000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075801.683509:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075801.683512:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075801.683514:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075801.683518:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2790 00000100:00080000:2.0:1769075801.683522:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075801.683525:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075801.683533:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075801.683547:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617184128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075801.683558:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075801.683562:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075801.683564:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075801.683571:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617184000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075801.683572:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2790 00000100:00080000:2.0:1769075801.683579:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075801.683588:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075801.683594:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075801.683606:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617184256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075801.683615:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075801.683618:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075801.683620:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075801.683623:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2790 00000100:00080000:2.0:1769075801.683627:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075801.683631:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075801.683636:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075801.683645:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617184384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075801.683654:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075801.683657:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075801.683659:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075801.683662:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2790 00000100:00080000:2.0:1769075801.683666:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769075801.683673:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075801.683675:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075801.683684:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075801.683706:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617184512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075801.683751:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075801/real 1769075801] req@ffff9b03482c5880 x1855007617184000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075801.683780:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617184000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075801.683796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075801.683803:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075801.683809:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617184000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075801.684051:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075801.684080:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075801.684094:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075801/real 1769075801] req@ffff9b03482c6680 x1855007617184128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075801.684107:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617184128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075801.684130:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075801.684134:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075801.684138:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075801.684144:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075801.684159:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075801.684168:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075801/real 1769075801] req@ffff9b03482c5c00 x1855007617184256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075801.684178:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617184256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075801.684186:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075801.684189:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075801.684192:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075801.684197:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075801.684212:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075801.684220:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075801/real 1769075801] req@ffff9b03482c4e00 x1855007617184384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075801.684231:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617184384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075801.684263:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075801.684266:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075801.684270:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075801.684286:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075801.684304:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075801.684314:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075801/real 1769075801] req@ffff9b03482c7b80 x1855007617184512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075801.684325:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617184512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075801.684333:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075801.684336:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075801.684339:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075802.195389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075802.579315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075802.579316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075802.579382:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075803.219318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075803.603327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075803.603350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075803.603379:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075804.243321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075804.627315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075804.627317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075804.627398:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075805.267312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075805.651379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075805.651416:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075805.651486:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075806.291317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075806.675315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075806.675315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075806.675443:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075806.803352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075806.803361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075806.803365:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075806.803373:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2795 00000100:00080000:2.0:1769075806.803383:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075806.803389:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075806.803422:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617184640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075806.803445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075806.803447:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075806.803449:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075806.803452:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2795 00000100:00080000:2.0:1769075806.803454:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075806.803456:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075806.803464:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075806.803474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617184768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075806.803483:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075806.803486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075806.803487:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075806.803490:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2795 00000100:00080000:2.0:1769075806.803492:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075806.803499:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075806.803503:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617184640:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769075806.803504:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075806.803517:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617184896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075806.803523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075806.803526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075806.803527:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075806.803531:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2795 00000100:00080000:2.0:1769075806.803534:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075806.803536:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075806.803541:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075806.803549:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617185024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075806.803555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075806.803566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075806.803567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075806.803570:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2795 00000100:00080000:2.0:1769075806.803573:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075806.803579:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075806.803584:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075806.803595:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617185152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075806.803649:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075806.803731:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075806/real 1769075806] req@ffff9b03482c5f80 x1855007617184640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075806.803831:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617184640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075806.803842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075806.803847:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075806.803850:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617184640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075806.803877:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075806.803900:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075806.803912:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075806/real 1769075806] req@ffff9b03482c6a00 x1855007617184768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075806.803922:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617184768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075806.803929:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075806.803932:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075806.803935:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075806.803940:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075806.803953:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075806.803959:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075806/real 1769075806] req@ffff9b03482c4000 x1855007617184896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075806.803968:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617184896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075806.803974:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075806.803976:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075806.803978:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617184896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075806.803983:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075806.803995:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075806.804001:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075806/real 1769075806] req@ffff9b03482c7100 x1855007617185024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075806.804010:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617185024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075806.804016:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075806.804019:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075806.804021:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075806.804025:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075806.804036:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075806.804045:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075806/real 1769075806] req@ffff9b03482c4700 x1855007617185152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075806.804058:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617185152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075806.804068:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075806.804072:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075806.804076:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075807.315371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075807.699381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075807.699396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075807.699507:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075808.339359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075808.723310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075808.723310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075808.723490:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075809.363346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075809.747293:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075809.747305:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075809.747389:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075810.387317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075810.771321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075810.771378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075810.771414:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075811.411351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075811.795340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075811.795340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075811.795430:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075811.923348:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075811.923360:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075811.923365:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075811.923375:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2801 00000100:00080000:3.0:1769075811.923385:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075811.923392:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075811.923431:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617185280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075811.923461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075811.923466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075811.923468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075811.923473:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2801 00000100:00080000:3.0:1769075811.923477:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075811.923482:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075811.923492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075811.923506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617185408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075811.923514:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617185280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075811.923522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075811.923528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075811.923530:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075811.923536:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2801 00000100:00080000:3.0:1769075811.923541:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075811.923546:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075811.923552:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075811.923566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617185536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075811.923578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075811.923582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075811.923586:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769075811.923588:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075811.923593:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2801 00000100:00080000:3.0:1769075811.923599:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075811.923604:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075811.923610:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075811.923623:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617185664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075811.923631:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075811.923635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769075811.923640:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075811/real 1769075811] req@ffff9b0352a1a680 x1855007617185280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075811.923640:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075811.923650:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2801 00000100:00080000:3.0:1769075811.923678:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075811.923684:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075811.923690:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075811.923696:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617185280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075811.923707:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617185792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075811.923712:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075811.923724:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075811.923731:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617185280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075811.923761:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075811.923789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075811.923805:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075811/real 1769075811] req@ffff9b0352a1b100 x1855007617185408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075811.923822:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617185408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075811.923832:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075811.923837:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075811.923842:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075811.923852:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075811.923877:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075811.923892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075811/real 1769075811] req@ffff9b0352a1ad80 x1855007617185536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075811.923908:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617185536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075811.923917:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075811.923922:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075811.923928:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075811.923938:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075811.923961:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075811.923972:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075811/real 1769075811] req@ffff9b0352a1b480 x1855007617185664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075811.923986:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617185664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075811.923996:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075811.924000:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075811.924005:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075811.924012:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075811.924032:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075811.924046:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075811/real 1769075811] req@ffff9b0352a18a80 x1855007617185792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075811.924061:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617185792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075811.924070:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075811.924075:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075811.924079:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617185792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075812.435358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075812.819326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075812.819330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075812.819391:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075813.459351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075813.843421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075813.843423:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075813.843603:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075814.483338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075814.867356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075814.867358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075814.867500:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075815.507366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075815.891394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075815.891399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075815.891531:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075816.531309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075816.915347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075816.915362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075816.915438:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075817.043363:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075817.043374:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075817.043389:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075817.043399:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2806 00000100:00080000:0.0:1769075817.043407:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075817.043414:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075817.043452:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617185920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075817.043510:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075817.043514:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075817.043516:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075817.043521:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2806 00000100:00080000:0.0:1769075817.043525:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075817.043529:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075817.043540:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075817.043554:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617186048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075817.043562:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617185920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075817.043600:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075817.043605:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075817.043608:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075817.043614:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2806 00000100:00080000:0.0:1769075817.043618:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075817.043621:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075817.043627:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075817.043640:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617186176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075817.043651:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075817.043654:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075817.043656:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075817.043659:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2806 00000800:00000100:3.0:1769075817.043666:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075817.043667:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075817.043687:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075817.043693:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075817.043705:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617186304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075817.043732:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075817.043735:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075817.043738:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075817.043743:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2806 00000100:00080000:0.0:1769075817.043747:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075817.043749:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075817/real 1769075817] req@ffff9b0365a39500 x1855007617185920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075817.043755:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075817.043764:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075817.043782:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617185920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075817.043783:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617186432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075817.043853:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075817.043861:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075817.043867:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617185920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075817.043893:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075817.043930:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075817.043942:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075817/real 1769075817] req@ffff9b0365a3bb80 x1855007617186048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075817.043956:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617186048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075817.043965:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075817.043968:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075817.043972:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075817.043977:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075817.044009:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075817.044022:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075817/real 1769075817] req@ffff9b0365a3b100 x1855007617186176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075817.044048:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617186176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075817.044070:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075817.044075:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075817.044080:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075817.044089:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075817.044115:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075817.044126:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075817/real 1769075817] req@ffff9b0365a3ad80 x1855007617186304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075817.044137:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617186304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075817.044144:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075817.044146:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075817.044149:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075817.044154:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075817.044166:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075817.044173:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075817/real 1769075817] req@ffff9b0365a39180 x1855007617186432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075817.044183:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617186432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075817.044195:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075817.044199:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075817.044203:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075817.555357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075817.939436:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075817.939452:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075817.939767:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075818.579312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075818.963315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075818.963315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075818.963488:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075819.603416:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075819.987388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075819.987396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075819.987485:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075820.627388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075821.011416:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075821.011416:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075821.011531:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075821.651392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075822.035352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075822.035427:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075822.035605:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075822.163336:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075822.163348:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075822.163352:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075822.163361:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2811 00000100:00080000:0.0:1769075822.163370:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075822.163376:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075822.163409:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617186560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075822.163429:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075822.163434:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075822.163436:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075822.163439:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2811 00000100:00080000:0.0:1769075822.163441:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075822.163444:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075822.163451:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075822.163460:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617186688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075822.163468:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617186560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075822.163471:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075822.163477:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075822.163479:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075822.163483:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2811 00000100:00080000:0.0:1769075822.163504:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075822.163507:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075822.163510:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075822.163519:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617186816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075822.163526:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075822.163529:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075822.163530:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075822.163533:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2811 00000100:00080000:0.0:1769075822.163542:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075822.163544:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075822.163549:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075822.163553:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075822.163560:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617186944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075822.163568:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075822.163571:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075822.163572:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075822.163575:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2811 00000100:00080000:0.0:1769075822.163577:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075822.163579:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075822.163582:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075822.163590:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617187072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075822.163599:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075822/real 1769075822] req@ffff9b0365a39180 x1855007617186560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075822.163619:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617186560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075822.163629:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075822.163634:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075822.163638:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617186560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075822.163661:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075822.163685:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075822.163698:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075822/real 1769075822] req@ffff9b0365a3ad80 x1855007617186688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075822.163712:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617186688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075822.163721:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075822.163724:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075822.163728:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075822.163735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075822.163755:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075822.163766:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075822/real 1769075822] req@ffff9b0365a3b100 x1855007617186816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075822.163778:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617186816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075822.163787:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075822.163790:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075822.163793:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075822.163800:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075822.163820:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075822.163833:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075822/real 1769075822] req@ffff9b0365a3bb80 x1855007617186944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075822.163846:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617186944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075822.163860:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075822.163863:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075822.163867:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617186944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075822.163873:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075822.163892:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075822.163902:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075822/real 1769075822] req@ffff9b0365a39500 x1855007617187072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075822.163915:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617187072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075822.163922:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075822.163926:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075822.163929:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075822.675348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075823.059346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075823.059375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075823.059497:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075823.699396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075824.083371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075824.083386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075824.083493:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075824.723350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075825.107343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075825.107350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075825.107457:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075825.747311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075826.131302:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075826.131319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075826.131375:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075826.771366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075827.155483:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075827.155557:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075827.155568:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769075827.283537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075827.283550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075827.283557:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075827.283571:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2816 00000100:00080000:3.0:1769075827.283611:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075827.283618:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075827.283669:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617187200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075827.283700:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075827.283705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075827.283708:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075827.283723:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2816 00000100:00080000:3.0:1769075827.283730:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075827.283736:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075827.283751:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075827.283757:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617187200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075827.283775:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617187328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075827.283797:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075827.283801:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075827.283804:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075827.283808:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2816 00000100:00080000:3.0:1769075827.283812:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075827.283816:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075827.283822:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075827.283835:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617187456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075827.283842:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075827.283871:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075827.283875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075827.283877:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075827.283881:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2816 00000100:00080000:3.0:1769075827.283885:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075827.283888:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075827.283893:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075827.283912:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075827/real 1769075827] req@ffff9b0352a19f80 x1855007617187200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075827.283912:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617187584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075827.283942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075827.283951:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075827.283953:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075827.283957:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2816 00000100:00080000:3.0:1769075827.283961:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075827.283969:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075827.283971:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617187200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769075827.283973:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1769075827.283990:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075827.284015:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075827.284019:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617187712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075827.284020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617187200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075827.284083:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075827.284110:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075827.284122:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075827/real 1769075827] req@ffff9b0352a1a300 x1855007617187328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075827.284134:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617187328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075827.284142:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075827.284145:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075827.284147:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075827.284152:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075827.284165:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075827.284173:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075827/real 1769075827] req@ffff9b0352a18e00 x1855007617187456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075827.284183:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617187456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075827.284201:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075827.284204:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075827.284206:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075827.284211:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075827.284224:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075827.284231:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075827/real 1769075827] req@ffff9b0352a18700 x1855007617187584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075827.284247:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617187584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075827.284254:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075827.284256:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075827.284259:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075827.284264:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075827.284307:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075827.284316:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075827/real 1769075827] req@ffff9b0352a1aa00 x1855007617187712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075827.284326:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617187712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075827.284333:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075827.284335:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075827.284338:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075827.795368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075828.179728:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769075828.179802:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075828.180014:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075828.819290:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075829.203331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075829.203399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075829.203416:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075829.843314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075830.227380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075830.227455:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075830.227461:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075830.867475:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075831.251422:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075831.252010:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075831.252487:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075831.891389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075832.275907:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075832.275941:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075832.276473:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075832.403530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075832.403544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075832.403550:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075832.403563:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2821 00000100:00080000:0.0:1769075832.403576:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075832.403585:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075832.403632:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617187840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075832.403685:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075832.403690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075832.403692:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075832.403698:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2821 00000100:00080000:0.0:1769075832.403702:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075832.403707:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075832.403720:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075832.403737:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617187968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075832.403747:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617187840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075832.403759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075832.403766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075832.403769:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075832.403775:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2821 00000100:00080000:0.0:1769075832.403779:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075832.403783:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075832.403790:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075832.403806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617188096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075832.403818:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075832.403823:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075832.403825:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075832.403830:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2821 00000100:00080000:0.0:1769075832.403836:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075832.403841:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075832.403848:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075832.403866:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617188224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075832.403871:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075832.403887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075832.403893:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075832.403895:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075832.403902:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2821 00000100:00080000:0.0:1769075832.403910:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075832.403915:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075832.403920:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075832.403934:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617188352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075832.403949:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075832/real 1769075832] req@ffff9b0365a3aa00 x1855007617187840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075832.403982:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617187840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075832.403997:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075832.404003:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075832.404009:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617187840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075832.404040:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075832.404072:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075832.404087:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075832/real 1769075832] req@ffff9b0365a3b480 x1855007617187968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075832.404109:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617187968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075832.404127:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075832.404134:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075832.404143:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617187968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075832.404155:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075832.404196:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075832.404218:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075832/real 1769075832] req@ffff9b0365a38e00 x1855007617188096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075832.404275:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617188096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075832.404287:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075832.404291:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075832.404295:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075832.404303:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075832.404358:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075832.404374:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075832/real 1769075832] req@ffff9b0365a39c00 x1855007617188224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075832.404390:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617188224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075832.404400:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075832.404404:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075832.404408:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075832.404416:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075832.404434:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075832.404446:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075832/real 1769075832] req@ffff9b0365a38a80 x1855007617188352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075832.404461:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617188352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075832.404470:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075832.404474:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075832.404477:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075832.915417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075833.299504:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075833.299817:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075833.299870:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075833.939354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075834.323369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075834.323454:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075834.323514:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075834.963347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075835.347385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075835.347385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075835.347475:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075835.987367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075836.371329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075836.371393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075836.371435:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075837.011393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075837.395393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075837.395408:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075837.395478:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075837.523435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075837.523449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075837.523455:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075837.523467:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2826 00000100:00080000:1.0:1769075837.523481:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075837.523507:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075837.523577:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617188480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075837.523624:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075837.523629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075837.523631:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075837.523637:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2826 00000100:00080000:1.0:1769075837.523642:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075837.523648:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075837.523662:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075837.523680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617188608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075837.523694:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075837.523697:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617188480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075837.523699:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075837.523703:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075837.523711:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2826 00000100:00080000:1.0:1769075837.523715:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075837.523718:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075837.523723:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075837.523745:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617188736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075837.523754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075837.523758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075837.523759:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075837.523762:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2826 00000100:00080000:1.0:1769075837.523766:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075837.523769:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075837.523773:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075837.523781:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617188864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075837.523787:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075837.523790:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075837.523792:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075837.523800:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2826 00000800:00000100:3.0:1769075837.523805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075837.523806:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075837.523815:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075837.523819:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075837.523831:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617188992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075837.523871:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075837/real 1769075837] req@ffff9b0343dec000 x1855007617188480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075837.523899:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617188480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075837.523921:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075837.523926:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075837.523932:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617188480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075837.523962:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075837.524006:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075837.524023:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075837/real 1769075837] req@ffff9b0343dec700 x1855007617188608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075837.524041:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617188608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075837.524053:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075837.524058:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075837.524062:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075837.524072:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075837.524099:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075837.524113:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075837/real 1769075837] req@ffff9b0343dedf80 x1855007617188736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075837.524130:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617188736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075837.524140:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075837.524144:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075837.524148:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075837.524157:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075837.524181:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075837.524194:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075837/real 1769075837] req@ffff9b0343ded180 x1855007617188864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075837.524210:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617188864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075837.524221:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075837.524225:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075837.524301:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075837.524311:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075837.524343:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075837.524369:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075837/real 1769075837] req@ffff9b0343dec380 x1855007617188992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075837.524391:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617188992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075837.524412:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075837.524417:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075837.524421:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617188992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075838.035341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075838.419384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075838.419392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075838.419525:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075839.059373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075839.443343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075839.443507:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075839.443533:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075840.083327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075840.467321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075840.467322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075840.467374:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075841.107355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075841.491329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075841.491331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075841.491399:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075842.131333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075842.515374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075842.515380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075842.515501:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075842.643375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075842.643398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075842.643404:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075842.643415:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2831 00000100:00080000:2.0:1769075842.643426:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075842.643433:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075842.643470:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617189120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075842.643514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075842.643519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075842.643522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075842.643529:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2831 00000100:00080000:2.0:1769075842.643535:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075842.643540:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075842.643559:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617189120:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769075842.643562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075842.643581:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617189248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075842.643591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075842.643595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769075842.643596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075842.643600:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2831 00000100:00080000:2.0:1769075842.643615:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075842.643619:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075842.643622:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075842.643634:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617189376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075842.643635:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769075842.643650:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075842.643654:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075842.643655:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075842.643659:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2831 00000100:00080000:2.0:1769075842.643662:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075842.643665:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075842.643669:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075842.643677:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617189504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075842.643684:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075842.643687:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075842.643689:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075842.643695:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2831 00000100:00100000:3.0:1769075842.643697:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075842/real 1769075842] req@ffff9b03482c5180 x1855007617189120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075842.643698:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075842.643705:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075842.643711:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075842.643736:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617189632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075842.643744:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617189120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075842.643763:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075842.643768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075842.643772:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617189120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075842.643795:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617189248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075842.643827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075842.643837:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075842/real 1769075842] req@ffff9b03482c6d80 x1855007617189248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075842.643847:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617189248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075842.643854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075842.643857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075842.643860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617189248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075842.643865:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617189376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075842.643877:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075842.643884:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075842/real 1769075842] req@ffff9b03482c6300 x1855007617189376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075842.643903:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617189376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075842.643909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075842.643911:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075842.643913:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617189376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075842.643918:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617189504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075842.643929:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075842.643936:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075842/real 1769075842] req@ffff9b03482c4a80 x1855007617189504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075842.643944:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617189504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075842.643951:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075842.643953:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075842.643955:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617189504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075842.643959:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617189632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075842.643969:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075842.643975:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075842/real 1769075842] req@ffff9b03482c4380 x1855007617189632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075842.643984:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617189632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075842.643990:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075842.643992:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075842.643995:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617189632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075843.155388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075843.539338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075843.539343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075843.539380:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075844.179371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075844.563385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075844.563391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075844.563525:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075845.203333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075845.587345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075845.587373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075845.587454:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075846.227389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075846.611369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075846.611370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075846.611431:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075847.251304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075847.635345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075847.635382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075847.635502:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075847.763390:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075847.763404:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075847.763411:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075847.763424:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2836 00000100:00080000:3.0:1769075847.763435:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075847.763445:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075847.763492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617189760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075847.763530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075847.763536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075847.763552:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075847.763558:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2836 00000100:00080000:3.0:1769075847.763563:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075847.763570:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075847.763582:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075847.763599:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617189888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075847.763613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075847.763618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075847.763621:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075847.763627:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2836 00000100:00080000:3.0:1769075847.763633:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075847.763642:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617189760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075847.763643:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075847.763658:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075847.763679:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617190016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075847.763693:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075847.763698:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075847.763701:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075847.763706:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2836 00000100:00080000:3.0:1769075847.763711:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075847.763715:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075847.763722:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075847.763740:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617190144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075847.763754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075847.763759:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075847.763762:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075847.763770:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2836 00000100:00080000:3.0:1769075847.763777:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075847.763790:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075847.763797:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075847.763815:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617189888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075847.763818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617190272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075847.763891:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075847.763933:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075847.763968:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075848.275306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075848.659325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075848.659327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075848.659403:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075849.299297:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075849.683361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075849.683385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075849.683478:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075850.323314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075850.707330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075850.707360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075850.707374:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769075850.835474:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769075850.835515:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769075850.835534:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769075850.835584:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769075850.835594:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769075850.835615:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769075850.835628:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769075850.835666:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075847/real 1769075850] req@ffff9b0352a18e00 x1855007617190016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075850.835698:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617190016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075850.835713:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075850.835719:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075850.835724:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075850.835749:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075847/real 1769075850] req@ffff9b0352a1a300 x1855007617190144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075850.835763:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617190144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075850.835772:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075850.835777:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075850.835781:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075850.835788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075847/real 1769075850] req@ffff9b0352a19f80 x1855007617190272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075850.835801:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617190272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075850.835809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075850.835814:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075850.835818:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075850.835825:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075847/real 1769075850] req@ffff9b0352a1aa00 x1855007617189760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075850.835838:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617189760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075850.835845:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075850.835849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075850.835852:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617189760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075850.835860:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075847/real 1769075850] req@ffff9b0352a18700 x1855007617189888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075850.835873:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617189888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075850.835881:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075850.835884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075850.835888:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617189888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075851.347364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075851.731392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075851.731392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075851.731527:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075852.371347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075852.755370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075852.755370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075852.755522:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075852.883356:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075852.883366:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075852.883371:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075852.883383:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2842 00000100:00080000:3.0:1769075852.883395:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075852.883403:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075852.883459:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617190400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075852.883493:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075852.883499:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075852.883502:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075852.883508:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2842 00000100:00080000:3.0:1769075852.883514:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075852.883519:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075852.883530:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075852.883563:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617190528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075852.883570:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617190400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075852.883577:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075852.883581:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075852.883584:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075852.883588:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2842 00000100:00080000:3.0:1769075852.883593:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075852.883597:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075852.883602:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075852.883613:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617190656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075852.883623:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075852.883627:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075852.883629:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075852.883633:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2842 00000100:00080000:3.0:1769075852.883637:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769075852.883638:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075852.883641:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075852.883646:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075852.883657:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617190784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075852.883667:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075852.883670:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075852.883672:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075852.883676:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2842 00000100:00100000:1.0:1769075852.883680:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075852/real 1769075852] req@ffff9b0352a18a80 x1855007617190400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075852.883680:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075852.883684:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075852.883690:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075852.883700:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617190912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075852.883701:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617190400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075852.883716:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075852.883721:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075852.883726:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617190400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075852.883753:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075852.883781:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075852.883797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075852/real 1769075852] req@ffff9b0352a1b480 x1855007617190528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075852.883813:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617190528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075852.883825:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075852.883829:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075852.883834:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075852.883842:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075852.883867:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075852.883881:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075852/real 1769075852] req@ffff9b0352a1ad80 x1855007617190656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075852.883895:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617190656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075852.883905:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075852.883909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075852.883913:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075852.883920:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075852.883941:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075852.883953:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075852/real 1769075852] req@ffff9b0352a1b100 x1855007617190784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075852.883968:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617190784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075852.884017:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075852.884022:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075852.884027:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075852.884035:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075852.884060:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075852.884074:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075852/real 1769075852] req@ffff9b0352a1a680 x1855007617190912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075852.884089:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617190912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075852.884099:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075852.884103:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075852.884107:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617190912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075853.395329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075853.779330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075853.779335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075853.779430:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075854.419296:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075854.803318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075854.803321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075854.803373:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075855.443303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075855.827326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075855.827326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075855.827393:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075856.467315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075856.851362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075856.851400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075856.851446:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075857.491314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075857.875373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075857.875377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075857.875447:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075858.003366:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075858.003379:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075858.003385:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075858.003399:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2847 00000100:00080000:3.0:1769075858.003412:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075858.003421:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075858.003482:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617191040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075858.003533:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075858.003540:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075858.003543:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075858.003550:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2847 00000100:00080000:3.0:1769075858.003556:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075858.003563:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075858.003576:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075858.003613:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617191040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075858.003616:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617191168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075858.003636:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075858.003641:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075858.003644:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075858.003650:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2847 00000100:00080000:3.0:1769075858.003657:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075858.003663:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075858.003682:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769075858.003686:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075858.003724:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617191296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075858.003748:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075858.003753:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075858.003757:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075858.003765:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2847 00000100:00080000:3.0:1769075858.003771:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075858.003774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075858/real 1769075858] req@ffff9b0352a18000 x1855007617191040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075858.003781:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075858.003792:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075858.003819:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617191424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075858.003824:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617191040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075858.003857:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075858.003864:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075858.003869:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075858.003873:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075858.003879:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2847 00000100:00080000:1.0:1769075858.003921:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769075858.003922:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075858.003929:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617191040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075858.003935:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075858.003954:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075858.003973:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617191552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075858.003977:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075858.004017:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075858.004040:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075858/real 1769075858] req@ffff9b0352a1b800 x1855007617191168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075858.004053:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617191168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075858.004062:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075858.004065:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075858.004068:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075858.004074:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075858.004090:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075858.004098:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075858/real 1769075858] req@ffff9b0352a18380 x1855007617191296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075858.004109:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617191296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075858.004116:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075858.004119:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075858.004122:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075858.004127:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075858.004140:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075858.004148:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075858/real 1769075858] req@ffff9b0352a19c00 x1855007617191424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075858.004174:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617191424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075858.004181:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075858.004183:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075858.004186:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075858.004199:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075858.004212:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075858.004220:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075858/real 1769075858] req@ffff9b0352a19500 x1855007617191552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075858.004247:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617191552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075858.004261:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075858.004296:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075858.004299:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075858.515290:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075858.899368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075858.899386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075858.899471:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075859.539348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075859.923375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075859.923386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075859.923487:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075860.563367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075860.947374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075860.947396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075860.947519:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075861.587349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075861.971384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075861.971394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075861.971534:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075862.611355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075862.995350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075862.995386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075862.995557:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075863.123435:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075863.123448:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075863.123454:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075863.123466:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2852 00000100:00080000:1.0:1769075863.123479:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075863.123489:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075863.123536:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617191680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075863.123576:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075863.123584:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075863.123587:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075863.123595:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2852 00000100:00080000:1.0:1769075863.123602:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075863.123609:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075863.123625:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075863.123649:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617191808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075863.123649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617191680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075863.123671:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075863.123677:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075863.123681:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075863.123687:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2852 00000100:00080000:1.0:1769075863.123693:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075863.123700:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075863.123709:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075863.123725:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617191936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075863.123741:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769075863.123743:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075863.123747:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075863.123751:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075863.123757:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2852 00000100:00080000:1.0:1769075863.123763:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075863.123769:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075863.123777:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075863.123793:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617192064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075863.123834:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075863/real 1769075863] req@ffff9b0343dee300 x1855007617191680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075863.123845:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075863.123852:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075863.123855:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075863.123862:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2852 00000100:00080000:1.0:1769075863.123869:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075863.123870:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617191680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075863.123876:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075863.123885:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1769075863.123889:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075863.123897:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075863.123904:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617192192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075863.123905:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617191680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075863.123942:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075863.123989:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075863.124014:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075863/real 1769075863] req@ffff9b0343def100 x1855007617191808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075863.124031:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617191808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075863.124042:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075863.124046:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075863.124050:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075863.124059:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075863.124083:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075863.124095:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075863/real 1769075863] req@ffff9b0343ded880 x1855007617191936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075863.124110:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617191936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075863.124119:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075863.124123:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075863.124127:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617191936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075863.124134:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075863.124152:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075863.124162:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075863/real 1769075863] req@ffff9b0343def800 x1855007617192064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075863.124177:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617192064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075863.124186:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075863.124190:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075863.124193:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075863.124200:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075863.124216:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075863.124226:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075863/real 1769075863] req@ffff9b0343deea00 x1855007617192192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075863.124278:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617192192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075863.124289:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075863.124292:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075863.124296:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075863.635394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075864.019373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075864.019419:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075864.019496:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075864.659386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075865.043381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075865.043414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075865.043500:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075865.683323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075866.067373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075866.067388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075866.067457:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075866.707336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075867.091314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075867.091339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075867.091409:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075867.731337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075868.115339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075868.115339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075868.115382:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075868.243304:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075868.243312:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075868.243316:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075868.243325:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2857 00000100:00080000:1.0:1769075868.243334:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075868.243339:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075868.243371:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617192320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075868.243395:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075868.243398:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075868.243400:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075868.243404:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2857 00000100:00080000:1.0:1769075868.243418:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075868.243422:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075868.243431:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075868.243446:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617192448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075868.243457:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075868.243459:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617192320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075868.243460:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075868.243462:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075868.243466:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2857 00000100:00080000:1.0:1769075868.243479:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075868.243484:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075868.243488:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075868.243500:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617192576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075868.243508:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075868.243511:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075868.243513:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075868.243517:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2857 00000100:00080000:1.0:1769075868.243521:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769075868.243522:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075868.243535:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075868.243539:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075868.243548:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617192704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075868.243556:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075868.243559:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075868.243561:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075868.243565:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2857 00000100:00080000:1.0:1769075868.243568:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075868.243572:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075868.243572:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075868/real 1769075868] req@ffff9b0343deed80 x1855007617192320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769075868.243576:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075868.243584:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617192832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075868.243590:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617192320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075868.243609:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075868.243614:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075868.243618:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617192320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075868.243642:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075868.243662:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075868.243672:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075868/real 1769075868] req@ffff9b0343deca80 x1855007617192448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075868.243680:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617192448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075868.243686:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075868.243688:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075868.243691:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075868.243695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075868.243705:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075868.243710:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075868/real 1769075868] req@ffff9b0343dece00 x1855007617192576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075868.243750:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617192576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075868.243770:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075868.243773:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075868.243776:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075868.243783:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075868.243805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075868.243816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075868/real 1769075868] req@ffff9b0343defb80 x1855007617192704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075868.243827:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617192704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075868.243833:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075868.243835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075868.243837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075868.243841:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075868.243855:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075868.243861:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075868/real 1769075868] req@ffff9b0343dee680 x1855007617192832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075868.243869:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617192832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075868.243874:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075868.243875:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075868.243877:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617192832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075868.755336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075869.139353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075869.139353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075869.139425:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075869.779307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075870.163321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075870.163321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075870.163404:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075870.803372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075871.187379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075871.187397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075871.187440:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075871.827353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075872.211361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075872.211371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075872.211474:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075872.851367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075873.235348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075873.235448:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075873.235583:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075873.363403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075873.363424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075873.363432:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075873.363451:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2862 00000100:00080000:1.0:1769075873.363470:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075873.363486:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075873.363563:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617192960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075873.363623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075873.363652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075873.363656:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075873.363664:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2862 00000100:00080000:1.0:1769075873.363672:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075873.363680:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075873.363702:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075873.363720:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617192960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075873.363729:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617193088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075873.363749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075873.363755:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075873.363759:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075873.363766:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2862 00000100:00080000:1.0:1769075873.363771:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075873.363779:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075873.363787:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075873.363805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617193216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075873.363820:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075873.363826:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075873.363829:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075873.363835:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2862 00000800:00000100:3.0:1769075873.363840:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075873.363842:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075873.363848:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075873.363857:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075873.363892:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617193344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075873.363910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075873.363914:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075873/real 1769075873] req@ffff9b0343def480 x1855007617192960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075873.363917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075873.363922:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075873.363930:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2862 00000100:00080000:1.0:1769075873.363936:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075873.363943:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075873.363950:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617192960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769075873.363952:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075873.363967:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617193472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075873.363972:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075873.363981:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075873.363989:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617192960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075873.364033:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075873.364078:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075873.364098:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075873/real 1769075873] req@ffff9b0343ded500 x1855007617193088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075873.364115:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617193088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075873.364128:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075873.364132:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075873.364137:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075873.364144:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075873.364167:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075873.364178:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075873/real 1769075873] req@ffff9b0343dee680 x1855007617193216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075873.364202:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617193216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075873.364212:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075873.364216:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075873.364220:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075873.364226:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075873.364285:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075873.364299:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075873/real 1769075873] req@ffff9b0343defb80 x1855007617193344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075873.364325:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617193344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075873.364335:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075873.364340:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075873.364344:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075873.364350:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075873.364369:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075873.364380:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075873/real 1769075873] req@ffff9b0343dece00 x1855007617193472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075873.364394:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617193472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075873.364412:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075873.364416:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075873.364420:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075873.875370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075874.259361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075874.259391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075874.259559:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075874.899414:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075875.283353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075875.283359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075875.283446:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075875.923347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075876.307348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075876.307349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075876.307409:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075876.947389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075877.331457:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075877.331486:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075877.331495:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075877.971373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075878.355364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075878.355369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075878.355562:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769075878.483508:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075878.483527:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769075878.483532:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075878.483566:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2867 00000100:00080000:2.0:1769075878.483593:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075878.483601:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769075878.483657:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617193600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075878.483692:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075878.483697:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769075878.483699:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075878.483704:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2867 00000100:00080000:2.0:1769075878.483709:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075878.483714:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075878.483727:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075878.483743:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617193728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075878.483760:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075878.483770:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1769075878.483773:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617193600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769075878.483775:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075878.483802:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2867 00000100:00080000:2.0:1769075878.483807:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075878.483811:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075878.483818:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075878.483838:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617193856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075878.483850:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075878.483854:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769075878.483856:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075878.483862:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2867 00000100:00080000:2.0:1769075878.483890:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075878.483895:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075878.483902:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075878.483918:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075878.483920:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617193984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075878.483945:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769075878.483948:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769075878.483951:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769075878.483955:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2867 00000100:00080000:2.0:1769075878.483963:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769075878.483969:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769075878.483973:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075878.483993:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617194112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075878.484065:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075878/real 1769075878] req@ffff9b03482c5500 x1855007617193600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075878.484096:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617193600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075878.484114:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075878.484122:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075878.484129:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617193600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075878.484175:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075878.484220:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075878.484238:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075878/real 1769075878] req@ffff9b03482c4380 x1855007617193728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075878.484254:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617193728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075878.484299:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075878.484304:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075878.484308:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075878.484316:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075878.484341:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075878.484353:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075878/real 1769075878] req@ffff9b03482c4a80 x1855007617193856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075878.484367:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617193856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075878.484376:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075878.484380:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075878.484400:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075878.484407:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075878.484427:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075878.484436:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075878/real 1769075878] req@ffff9b03482c6300 x1855007617193984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075878.484448:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617193984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075878.484457:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075878.484460:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075878.484463:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617193984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075878.484469:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617194112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075878.484484:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075878.484492:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075878/real 1769075878] req@ffff9b03482c6d80 x1855007617194112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075878.484502:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617194112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075878.484511:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075878.484514:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075878.484517:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617194112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075878.995426:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075879.379436:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075879.379444:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075879.379587:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075880.019357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075880.403425:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075880.403425:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075880.403602:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075881.043430:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075881.427384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075881.427450:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075881.427469:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075882.067409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075882.451471:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075882.451472:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075882.451528:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075883.091394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075883.475362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075883.475362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075883.475561:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075883.603394:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075883.603408:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075883.603415:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075883.603430:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2872 00000100:00080000:3.0:1769075883.603445:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075883.603479:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075883.603538:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617194240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075883.603574:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075883.603579:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075883.603582:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075883.603591:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2872 00000100:00080000:3.0:1769075883.603600:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075883.603610:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075883.603628:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075883.603630:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617194240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075883.603671:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617194368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075883.603699:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075883.603707:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075883.603710:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075883.603719:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2872 00000100:00080000:3.0:1769075883.603727:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769075883.603737:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075883.603739:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075883.603754:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075883.603812:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075883/real 1769075883] req@ffff9b0352a19500 x1855007617194240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075883.603836:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617194240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075883.603850:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075883.603855:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075883.603860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617194240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075883.603904:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617194368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075883.603915:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617194496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075883.603945:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075883.603961:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075883/real 1769075883] req@ffff9b0352a19c00 x1855007617194368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075883.603976:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617194368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075883.603986:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075883.603990:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075883.603993:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617194368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075883.604343:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617194496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075883.604402:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075883.604434:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075883.604441:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769075883.604444:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075883/real 1769075883] req@ffff9b0352a18380 x1855007617194496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075883.604446:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075883.604477:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2872 00000100:00080000:3.0:1769075883.604489:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075883.604498:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075883.604505:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617194496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769075883.604507:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769075883.604525:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075883.604532:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075883.604538:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617194624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075883.604542:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617194496:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075883.604567:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075883.604571:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769075883.604575:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617194624:192.168.204.160@tcp:8:kworker.0 00000100:00080000:3.0:1769075883.604576:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075883.604591:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2872 00000100:00080000:3.0:1769075883.604599:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075883.604608:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769075883.604616:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769075883.604618:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075883.604636:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075883/real 1769075883] req@ffff9b0352a1b800 x1855007617194624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075883.604650:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617194752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075883.604659:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617194624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075883.604693:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075883.604700:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075883.604706:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617194624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075883.604721:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617194752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075883.604742:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075883.604752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075883/real 1769075883] req@ffff9b0352a18000 x1855007617194752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075883.604765:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617194752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075883.604774:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075883.604777:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075883.604780:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617194752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075884.115364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075884.499323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075884.499361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075884.499466:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075885.139419:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075885.523385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075885.523440:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075885.523443:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075886.163465:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075886.547312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075886.547358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075886.547395:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075887.187377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075887.571377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075887.571426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075887.571499:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075888.211340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075888.595406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075888.595481:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075888.595511:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075888.723410:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075888.723421:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075888.723426:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075888.723438:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2877 00000100:00080000:3.0:1769075888.723450:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075888.723457:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075888.723494:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617194880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075888.723525:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075888.723528:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075888.723530:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075888.723534:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2877 00000100:00080000:3.0:1769075888.723537:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075888.723540:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075888.723549:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075888.723559:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617195008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075888.723569:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075888.723573:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075888.723575:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075888.723578:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2877 00000100:00080000:3.0:1769075888.723581:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075888.723585:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075888.723591:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075888.723598:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617194880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075888.723603:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617195136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075888.723612:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075888.723616:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075888.723620:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075888.723623:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2877 00000100:00080000:3.0:1769075888.723626:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075888.723628:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075888.723632:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075888.723642:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617195264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075888.723651:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075888.723654:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075888.723656:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075888.723659:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2877 00000100:00080000:3.0:1769075888.723662:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075888.723665:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075888.723668:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769075888.723686:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075888.723693:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617195392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075888.723730:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075888/real 1769075888] req@ffff9b0352a1a680 x1855007617194880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075888.723749:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617194880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075888.723759:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075888.723764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075888.723767:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617194880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075888.723774:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075888.723792:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075888.723801:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075888/real 1769075888] req@ffff9b0352a1b100 x1855007617195008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075888.723810:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617195008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075888.723817:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075888.723820:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075888.723823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075888.723842:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075888.723855:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075888.723862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075888/real 1769075888] req@ffff9b0352a1ad80 x1855007617195136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075888.723871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617195136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075888.723883:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075888.723886:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075888.723889:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075888.723893:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075888.723905:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075888.723911:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075888/real 1769075888] req@ffff9b0352a1b480 x1855007617195264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075888.723920:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617195264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075888.723926:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075888.723928:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075888.723930:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075888.723935:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075888.723946:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075888.723952:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075888/real 1769075888] req@ffff9b0352a18a80 x1855007617195392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075888.723961:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617195392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075888.723967:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075888.723969:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075888.723973:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075889.235413:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075889.619427:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075889.619466:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075889.619510:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075890.259460:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075890.643455:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075890.643464:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075890.643604:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075891.283339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075891.667377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075891.667381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075891.667411:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075892.307356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075892.691389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075892.691391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075892.691500:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075893.331376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075893.715404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075893.715445:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075893.715529:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075893.843443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075893.843454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075893.843459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075893.843470:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2883 00000100:00080000:0.0:1769075893.843481:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075893.843488:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075893.843530:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617195520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075893.843565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075893.843569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075893.843571:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075893.843575:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2883 00000100:00080000:0.0:1769075893.843579:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075893.843582:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075893.843592:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075893.843604:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617195648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075893.843615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075893.843620:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075893.843623:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075893.843626:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2883 00000100:00080000:0.0:1769075893.843630:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075893.843633:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075893.843638:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075893.843654:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617195776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075893.843656:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617195520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075893.843686:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075893.843690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075893.843691:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075893.843696:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2883 00000100:00080000:0.0:1769075893.843699:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075893.843703:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075893.843709:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075893.843721:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617195904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075893.843730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075893.843734:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075893.843736:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075893.843745:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2883 00000800:00000100:1.0:1769075893.843746:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075893.843750:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075893.843757:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075893.843763:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075893.843776:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617196032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075893.843848:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075893/real 1769075893] req@ffff9b0365a38a80 x1855007617195520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075893.843877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617195520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075893.843892:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075893.843916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075893.843921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617195520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075893.843930:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075893.843959:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075893.843973:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075893/real 1769075893] req@ffff9b0365a39c00 x1855007617195648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075893.843985:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617195648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075893.843994:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075893.843997:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075893.844002:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075893.844027:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075893.844043:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075893.844053:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075893/real 1769075893] req@ffff9b0365a38e00 x1855007617195776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075893.844064:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617195776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075893.844072:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075893.844075:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075893.844078:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075893.844083:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075893.844114:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075893.844122:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075893/real 1769075893] req@ffff9b0365a3b480 x1855007617195904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075893.844132:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617195904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075893.844139:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075893.844141:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075893.844144:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617195904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075893.844149:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075893.844161:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075893.844169:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075893/real 1769075893] req@ffff9b0365a3aa00 x1855007617196032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075893.844180:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617196032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075893.844187:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075893.844189:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075893.844192:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075894.355372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075894.739479:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075894.739537:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075894.739689:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075895.379369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075895.763431:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075895.763670:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075895.763697:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075896.403377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075896.787399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075896.787406:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075896.787568:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075897.427315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075897.811348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075897.811606:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075897.811651:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075898.451350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075898.835341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075898.835351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075898.835678:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075898.963466:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075898.963478:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075898.963483:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075898.963493:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2888 00000100:00080000:3.0:1769075898.963502:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075898.963507:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075898.963560:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617196160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075898.963584:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075898.963587:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075898.963588:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075898.963591:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2888 00000100:00080000:3.0:1769075898.963594:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075898.963597:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075898.963604:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075898.963612:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617196288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075898.963620:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075898.963623:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075898.963624:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075898.963626:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2888 00000100:00080000:3.0:1769075898.963628:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075898.963630:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075898.963633:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075898.963642:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617196416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075898.963649:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075898.963652:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075898.963654:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075898.963656:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2888 00000100:00080000:3.0:1769075898.963658:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075898.963660:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075898.963664:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075898.963673:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617196544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075898.963685:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075898.963691:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075898.963695:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075898.963696:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617196160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075898.963704:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2888 00000100:00080000:3.0:1769075898.963720:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075898.963725:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075898.963728:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075898.963737:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617196672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075898.963832:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075898.963905:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075898/real 1769075898] req@ffff9b0352a18700 x1855007617196160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075898.963930:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617196160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075898.963945:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075898.963951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075898.963956:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617196160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075898.963966:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075898.963993:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075898.964008:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075898/real 1769075898] req@ffff9b0352a1aa00 x1855007617196288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075898.964023:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617196288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075898.964034:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075898.964038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075898.964042:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075898.964049:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075898.964067:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075898.964078:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075898/real 1769075898] req@ffff9b0352a19f80 x1855007617196416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075898.964091:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617196416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075898.964100:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075898.964103:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075898.964107:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075898.964144:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075898.964164:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075898.964175:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075898/real 1769075898] req@ffff9b0352a1a300 x1855007617196544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075898.964328:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617196544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075898.964354:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075898.964360:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075898.964365:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075898.964374:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075898.964405:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075898.964420:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075898/real 1769075898] req@ffff9b0352a18e00 x1855007617196672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075898.964434:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617196672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075898.964444:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075898.964448:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075898.964451:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075899.475443:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075899.859372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075899.859372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075899.859677:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075900.499348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075900.883390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075900.883390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075900.883472:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075901.523334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075901.907363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075901.907373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075901.907488:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075902.547312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075902.931350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075902.931361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075902.931449:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075903.571353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075903.955312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769075903.955351:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075903.955382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769075904.083316:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075904.083327:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075904.083333:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075904.083343:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2893 00000100:00080000:3.0:1769075904.083353:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075904.083360:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075904.083397:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617196800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075904.083431:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075904.083435:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075904.083437:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075904.083442:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2893 00000100:00080000:3.0:1769075904.083446:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075904.083450:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075904.083460:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075904.083475:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617196928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075904.083479:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617196800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075904.083495:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075904.083499:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075904.083502:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075904.083507:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2893 00000100:00080000:3.0:1769075904.083512:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075904.083517:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075904.083522:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075904.083536:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617197056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075904.083540:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075904.083552:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075904.083557:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075904.083559:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075904.083565:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2893 00000100:00080000:3.0:1769075904.083570:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075904.083574:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075904.083578:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075904/real 1769075904] req@ffff9b0352a18e00 x1855007617196800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769075904.083582:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075904.083602:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617196800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075904.083603:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617197184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075904.083636:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075904.083639:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075904.083642:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075904.083645:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769075904.083647:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769075904.083651:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617196800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075904.083656:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2893 00000100:00080000:3.0:1769075904.083661:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075904.083666:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075904.083672:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075904.083674:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075904.083689:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617197312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769075904.083697:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075904.083710:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075904/real 1769075904] req@ffff9b0352a1a300 x1855007617196928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075904.083720:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617196928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075904.083728:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075904.083731:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075904.083734:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617196928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075904.083739:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075904.083750:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075904.083756:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075904/real 1769075904] req@ffff9b0352a19f80 x1855007617197056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075904.083763:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617197056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075904.083768:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075904.083770:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075904.083772:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075904.083776:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075904.083785:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075904.083790:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075904/real 1769075904] req@ffff9b0352a1aa00 x1855007617197184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075904.083797:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617197184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075904.083802:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075904.083804:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075904.083806:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075904.083814:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075904.083823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075904.083829:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075904/real 1769075904] req@ffff9b0352a18700 x1855007617197312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075904.083835:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617197312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075904.083840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075904.083841:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075904.083843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075904.595362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075904.979375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075904.979375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075904.979531:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075905.619359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075906.003302:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075906.003306:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075906.003459:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075906.643336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075907.027337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075907.027408:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075907.027464:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075907.667352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075908.051319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075908.051332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075908.051456:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075908.691334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075909.075407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075909.075421:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075909.075502:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075909.203457:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075909.203474:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075909.203479:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075909.203491:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2898 00000100:00080000:3.0:1769075909.203515:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075909.203523:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075909.203570:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617197440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075909.203621:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075909.203625:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075909.203627:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075909.203631:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2898 00000100:00080000:3.0:1769075909.203634:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075909.203637:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075909.203650:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075909.203669:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617197568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075909.203684:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617197440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075909.203690:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075909.203697:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075909.203700:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075909.203716:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2898 00000100:00080000:3.0:1769075909.203731:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075909.203736:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075909.203741:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075909.203756:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617197696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075909.203765:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075909.203767:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075909.203769:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075909.203773:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2898 00000100:00080000:3.0:1769075909.203776:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769075909.203782:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075909.203782:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075909.203794:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075909.203814:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617197824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075909.203821:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075909.203824:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075909.203825:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075909.203829:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2898 00000100:00080000:3.0:1769075909.203833:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075909.203836:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075909.203841:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075909.203851:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617197952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075909.203880:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075909/real 1769075909] req@ffff9b0352a18a80 x1855007617197440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075909.203902:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617197440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075909.203911:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075909.203915:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075909.203918:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617197440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075909.203939:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075909.203960:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075909.203969:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075909/real 1769075909] req@ffff9b0352a1b480 x1855007617197568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075909.203978:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617197568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075909.203984:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075909.203987:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075909.203989:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075909.203994:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075909.204005:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075909.204011:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075909/real 1769075909] req@ffff9b0352a1ad80 x1855007617197696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075909.204024:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617197696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075909.204046:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075909.204049:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075909.204051:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075909.204056:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075909.204089:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075909.204096:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075909/real 1769075909] req@ffff9b0352a1b100 x1855007617197824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075909.204106:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617197824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075909.204112:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075909.204115:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075909.204117:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075909.204121:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075909.204131:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075909.204137:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075909/real 1769075909] req@ffff9b0352a1a680 x1855007617197952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075909.204145:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617197952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075909.204155:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075909.204158:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075909.204160:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617197952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075909.715346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075910.099344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075910.099344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075910.099417:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075910.739358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075911.123358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075911.123433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075911.123501:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075911.763368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075912.147331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075912.147336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075912.147441:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075912.787333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075913.171340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075913.171359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075913.171499:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075913.811312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075914.195321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075914.195329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075914.195381:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075914.323311:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075914.323322:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075914.323327:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075914.323338:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2903 00000100:00080000:3.0:1769075914.323348:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075914.323355:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075914.323393:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617198080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075914.323421:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075914.323426:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075914.323428:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075914.323432:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2903 00000100:00080000:3.0:1769075914.323436:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075914.323441:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075914.323450:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075914.323463:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617198208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075914.323476:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769075914.323483:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617198080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075914.323483:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075914.323487:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075914.323497:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2903 00000100:00080000:3.0:1769075914.323502:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075914.323507:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075914.323512:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075914.323526:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617198336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075914.323536:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075914.323540:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075914.323542:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075914.323546:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2903 00000100:00080000:3.0:1769075914.323550:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075914.323554:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075914.323560:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075914.323571:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617198464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075914.323582:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075914.323586:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075914.323589:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075914.323594:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2903 00000100:00080000:3.0:1769075914.323598:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769075914.323601:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198208:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769075914.323605:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075914.323614:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075914.323629:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617198592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075914.323641:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075914.323669:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075914.323688:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075914.835306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075915.219356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075915.219357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075915.219428:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075915.859327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075916.243346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075916.243348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075916.243489:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075916.883312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075917.267362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075917.267366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075917.267462:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075917.395660:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769075917.395709:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075917.395728:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075917.395776:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075917.395786:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075917.395795:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075917.395804:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:3.0:1769075917.395948:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075914/real 1769075917] req@ffff9b0352a19500 x1855007617198592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075917.395980:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617198592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075917.395995:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075917.396001:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075917.396006:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075917.396016:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075914/real 1769075917] req@ffff9b0352a1b800 x1855007617198208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075917.396030:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617198208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075917.396040:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075917.396044:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075917.396047:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075917.396056:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075914/real 1769075917] req@ffff9b0352a18380 x1855007617198336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075917.396076:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617198336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075917.396085:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075917.396089:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075917.396093:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075917.396099:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075914/real 1769075917] req@ffff9b0352a19c00 x1855007617198464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075917.396112:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617198464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075917.396120:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075917.396123:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075917.396127:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075917.396134:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075914/real 1769075917] req@ffff9b0352a18000 x1855007617198080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075917.396147:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617198080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075917.396154:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075917.396158:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075917.396161:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617198080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075917.907376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075918.291307:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075918.291380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075918.291494:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075918.931697:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075919.315326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075919.315326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075919.315383:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075919.443336:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075919.443347:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075919.443352:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075919.443380:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2908 00000100:00080000:3.0:1769075919.443392:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075919.443401:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075919.443436:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617198720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075919.443462:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075919.443465:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075919.443466:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075919.443469:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2908 00000100:00080000:3.0:1769075919.443472:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075919.443475:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075919.443482:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075919.443490:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617198848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075919.443497:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075919.443499:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075919.443501:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075919.443503:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2908 00000100:00080000:3.0:1769075919.443505:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075919.443510:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075919.443515:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075919.443526:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617198976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075919.443536:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075919.443539:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075919.443541:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075919.443545:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2908 00000100:00100000:1.0:1769075919.443551:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617198720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075919.443552:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075919.443569:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075919.443579:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075919.443598:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617199104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075919.443624:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075919.443628:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075919.443631:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075919.443635:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2908 00000100:00080000:3.0:1769075919.443639:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769075919.443646:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075919.443646:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075919.443655:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075919.443669:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617199232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769075919.443697:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075919/real 1769075919] req@ffff9b0352a18000 x1855007617198720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075919.443728:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617198720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075919.443741:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075919.443745:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075919.443749:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617198720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075919.443758:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075919.443792:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075919.443812:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075919/real 1769075919] req@ffff9b0352a19c00 x1855007617198848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075919.443823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617198848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075919.443832:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075919.443837:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075919.443841:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075919.443871:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769075919.443906:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075919.443924:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075919/real 1769075919] req@ffff9b0352a18380 x1855007617198976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075919.443939:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617198976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075919.443948:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075919.443953:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075919.443957:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617198976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075919.443993:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075919.444024:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075919.444039:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075919/real 1769075919] req@ffff9b0352a1b800 x1855007617199104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075919.444055:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617199104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075919.444065:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075919.444070:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075919.444075:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075919.444084:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769075919.444108:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075919.444123:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075919/real 1769075919] req@ffff9b0352a19500 x1855007617199232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075919.444146:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617199232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075919.444156:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075919.444160:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769075919.444164:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075919.955293:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075920.339352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075920.339356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075920.339448:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075920.979321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075921.363313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075921.363313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075921.363387:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075922.003289:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075922.387294:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075922.387316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075922.387383:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075923.027284:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075923.411304:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075923.411359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075923.411376:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075924.051301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075924.435309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075924.435322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075924.435386:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075924.563383:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075924.563406:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075924.563412:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075924.563423:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2913 00000100:00080000:1.0:1769075924.563449:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075924.563456:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075924.563496:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617199360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075924.563544:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075924.563549:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075924.563551:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075924.563557:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2913 00000100:00080000:1.0:1769075924.563562:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075924.563596:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075924.563602:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617199360:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769075924.563608:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075924.563626:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617199488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075924.563639:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075924.563643:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075924.563645:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075924.563650:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2913 00000100:00080000:1.0:1769075924.563654:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075924.563659:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075924.563664:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769075924.563676:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075924.563689:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617199616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075924.563701:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075924.563704:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075924.563706:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075924.563711:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2913 00000100:00080000:1.0:1769075924.563716:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769075924.563722:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075924/real 1769075924] req@ffff9b0343deca80 x1855007617199360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075924.563723:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075924.563730:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075924.563743:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617199744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075924.563752:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769075924.563755:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617199360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075924.563757:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075924.563762:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075924.563791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075924.563796:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075924.563804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617199360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769075924.563805:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2913 00000100:00080000:1.0:1769075924.563814:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075924.563820:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075924.563826:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075924.563840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769075924.563841:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617199872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075924.563873:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075924.563889:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075924/real 1769075924] req@ffff9b0343deed80 x1855007617199488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075924.563903:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617199488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075924.563911:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075924.563916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075924.563920:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075924.563926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075924.563947:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075924.563958:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075924/real 1769075924] req@ffff9b0343deea00 x1855007617199616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075924.563971:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617199616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075924.563980:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075924.563983:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075924.563987:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075924.563993:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075924.564013:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075924.564024:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075924/real 1769075924] req@ffff9b0343def800 x1855007617199744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075924.564037:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617199744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075924.564046:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075924.564049:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075924.564053:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075924.564070:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075924.564089:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075924.564101:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075924/real 1769075924] req@ffff9b0343ded880 x1855007617199872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075924.564114:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617199872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075924.564123:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075924.564126:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075924.564130:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617199872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075925.075379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075925.459321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075925.459509:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075925.459578:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075926.099338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075926.483389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075926.483393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075926.483535:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075927.123368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075927.507300:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075927.507333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075927.507468:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075928.147335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075928.531393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075928.531393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075928.531517:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075929.171320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075929.555328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075929.555328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075929.555385:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075929.683383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075929.683391:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075929.683396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075929.683404:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2918 00000100:00080000:0.0:1769075929.683412:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075929.683418:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075929.683448:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617200000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075929.683477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075929.683479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075929.683481:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075929.683484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2918 00000100:00080000:0.0:1769075929.683486:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075929.683488:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075929.683495:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075929.683503:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617200128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075929.683510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075929.683513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075929.683515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075929.683517:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2918 00000100:00080000:0.0:1769075929.683519:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075929.683521:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075929.683525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075929.683532:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617200256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075929.683536:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617200000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075929.683543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075929.683547:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075929.683548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075929.683551:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2918 00000100:00080000:0.0:1769075929.683553:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075929.683555:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075929.683558:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075929.683566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617200384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075929.683571:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075929.683574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075929.683575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075929.683577:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2918 00000100:00080000:0.0:1769075929.683580:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075929.683582:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075929.683584:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075929.683592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617200512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075929.683602:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075929.683644:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075929/real 1769075929] req@ffff9b0365a39500 x1855007617200000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075929.683661:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617200000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075929.683670:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075929.683674:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075929.683678:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617200000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075929.683697:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075929.683716:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075929.683725:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075929/real 1769075929] req@ffff9b0365a3bb80 x1855007617200128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075929.683734:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617200128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075929.683740:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075929.683743:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075929.683745:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075929.683763:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075929.683774:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075929.683781:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075929/real 1769075929] req@ffff9b0365a3b100 x1855007617200256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075929.683789:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617200256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075929.683794:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075929.683795:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075929.683797:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075929.683801:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075929.683811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075929.683817:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075929/real 1769075929] req@ffff9b0365a3ad80 x1855007617200384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075929.683824:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617200384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075929.683830:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075929.683831:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075929.683833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075929.683837:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075929.683846:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075929.683851:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075929/real 1769075929] req@ffff9b0365a39180 x1855007617200512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075929.683858:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617200512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075929.683863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075929.683865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075929.683867:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075930.195425:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075930.579365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075930.579367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075930.579406:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075931.219355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075931.603364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075931.603401:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075931.603457:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075932.243321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075932.627359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075932.627434:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075932.627449:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075933.267377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075933.651385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075933.651443:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075933.651491:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075934.291304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075934.675375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075934.675396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075934.675471:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075934.803374:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075934.803386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075934.803391:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075934.803402:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2923 00000100:00080000:0.0:1769075934.803414:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075934.803422:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075934.803469:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617200640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075934.803493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075934.803497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075934.803498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075934.803502:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2923 00000100:00080000:0.0:1769075934.803504:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075934.803507:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075934.803527:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075934.803537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617200768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075934.803549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075934.803554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075934.803556:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075934.803561:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2923 00000100:00080000:0.0:1769075934.803565:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075934.803572:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075934.803579:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075934.803620:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617200640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075934.803622:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617200896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075934.803659:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075934.803661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075934.803663:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075934.803668:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2923 00000100:00080000:0.0:1769075934.803672:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075934.803676:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075934.803681:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075934.803692:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617201024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075934.803700:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075934.803703:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075934.803704:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075934.803707:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2923 00000100:00080000:0.0:1769075934.803710:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075934.803713:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075934.803716:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075934.803724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617201152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075934.803763:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075934.803840:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075934/real 1769075934] req@ffff9b0365a39180 x1855007617200640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075934.803891:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617200640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075934.803904:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075934.803909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075934.803914:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617200640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075934.803923:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075934.803948:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075934.803961:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075934/real 1769075934] req@ffff9b0365a3ad80 x1855007617200768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075934.803975:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617200768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075934.803985:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075934.803989:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075934.803993:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075934.804022:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075934.804043:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075934.804055:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075934/real 1769075934] req@ffff9b0365a3b100 x1855007617200896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075934.804069:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617200896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075934.804078:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075934.804082:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075934.804085:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617200896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075934.804092:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075934.804110:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075934.804119:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075934/real 1769075934] req@ffff9b0365a3bb80 x1855007617201024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075934.804133:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617201024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075934.804142:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075934.804146:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075934.804150:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075934.804157:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075934.804175:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075934.804196:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075934/real 1769075934] req@ffff9b0365a39500 x1855007617201152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075934.804217:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617201152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075934.804225:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075934.804228:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075934.804269:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075935.315360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075935.699348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075935.699369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075935.699392:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075936.339305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075936.723372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075936.723401:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075936.723461:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075937.363306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075937.747351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075937.747351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075937.747481:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075938.387440:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075938.771328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075938.771341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075938.771440:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075939.411366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075939.795362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075939.795434:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075939.795474:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075939.923502:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075939.923561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075939.923570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075939.923585:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2929 00000100:00080000:1.0:1769075939.923601:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075939.923613:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075939.923670:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617201280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075939.923712:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075939.923717:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075939.923720:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075939.923725:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2929 00000100:00080000:1.0:1769075939.923731:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075939.923736:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075939.923746:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075939.923761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617201408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075939.923776:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075939.923781:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075939.923784:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075939.923788:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2929 00000100:00080000:1.0:1769075939.923793:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075939.923797:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075939.923804:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075939.923812:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617201280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075939.923825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617201536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075939.923840:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075939.923845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075939.923847:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075939.923852:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2929 00000100:00080000:1.0:1769075939.923858:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075939.923863:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075939.923870:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075939.923885:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617201664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075939.923897:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075939.923902:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075939.923909:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769075939.923910:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769075939.923924:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2929 00000100:00080000:1.0:1769075939.923933:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075939.923938:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075939.923945:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075939.923960:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617201792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075939.923980:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075939/real 1769075939] req@ffff9b0343def100 x1855007617201280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075939.924013:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617201280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075939.924033:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075939.924041:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075939.924049:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617201280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075939.924065:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075939.924108:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075939.924133:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075939/real 1769075939] req@ffff9b0343dee300 x1855007617201408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075939.924158:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617201408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075939.924170:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075939.924174:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075939.924178:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075939.924209:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075939.924277:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075939.924294:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075939/real 1769075939] req@ffff9b0343dec380 x1855007617201536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075939.924309:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617201536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075939.924320:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075939.924324:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075939.924328:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075939.924335:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075939.924358:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075939.924369:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075939/real 1769075939] req@ffff9b0343ded180 x1855007617201664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075939.924384:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617201664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075939.924393:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075939.924397:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075939.924400:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075939.924407:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075939.924425:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075939.924435:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075939/real 1769075939] req@ffff9b0343dedf80 x1855007617201792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075939.924449:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617201792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075939.924458:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075939.924462:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075939.924465:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617201792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075940.435417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075940.819358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075940.819371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075940.819434:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075941.459340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075941.843343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075941.843381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075941.843474:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075942.483363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075942.867370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075942.867407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075942.867587:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075943.507322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075943.891382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075943.891382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075943.891470:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075944.531414:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075944.915329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075944.915351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075944.915505:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075945.043388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075945.043401:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075945.043413:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075945.043448:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2934 00000100:00080000:0.0:1769075945.043462:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075945.043470:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075945.043526:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617201920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075945.043560:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075945.043564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075945.043565:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075945.043569:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2934 00000100:00080000:0.0:1769075945.043572:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075945.043575:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075945.043585:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075945.043596:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617202048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075945.043605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075945.043608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075945.043609:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075945.043614:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2934 00000100:00080000:0.0:1769075945.043616:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075945.043619:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075945.043623:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075945.043635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617202176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075945.043635:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617201920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075945.043663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075945.043666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075945.043668:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075945.043673:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2934 00000100:00080000:0.0:1769075945.043677:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075945.043679:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075945.043685:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075945.043695:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617202304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075945.043702:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075945.043704:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075945.043706:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075945.043712:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2934 00000100:00080000:0.0:1769075945.043714:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075945.043718:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075945.043724:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075945.043738:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075945.043745:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617202432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075945.043806:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075945/real 1769075945] req@ffff9b0365a3aa00 x1855007617201920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075945.043830:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617201920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075945.043842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075945.043848:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075945.043852:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617201920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075945.043863:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075945.043890:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075945.043902:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075945/real 1769075945] req@ffff9b0365a3b480 x1855007617202048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075945.043915:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617202048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075945.043924:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075945.043928:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075945.043931:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075945.043959:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075945.043976:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075945.043986:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075945/real 1769075945] req@ffff9b0365a38e00 x1855007617202176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075945.043999:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617202176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075945.044007:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075945.044010:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075945.044014:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075945.044020:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075945.044035:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075945.044043:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075945/real 1769075945] req@ffff9b0365a39c00 x1855007617202304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075945.044068:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617202304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075945.044077:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075945.044081:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075945.044084:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075945.044090:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075945.044107:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075945.044117:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075945/real 1769075945] req@ffff9b0365a38a80 x1855007617202432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075945.044128:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617202432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075945.044144:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075945.044147:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075945.044151:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075945.555347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075945.939429:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075945.939433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075945.939477:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075946.579335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075946.963344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075946.963406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075946.963465:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075947.603314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075947.987370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075947.987371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075947.987480:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075948.627305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075949.011381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075949.011403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075949.011493:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075949.651437:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075950.035383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075950.035414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075950.035545:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075950.163501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075950.163515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075950.163522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075950.163534:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2939 00000100:00080000:1.0:1769075950.163547:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075950.163556:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075950.163623:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617202560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075950.163661:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075950.163665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075950.163668:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075950.163673:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2939 00000100:00080000:1.0:1769075950.163678:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075950.163682:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075950.163693:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769075950.163708:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617202688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075950.163723:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075950.163728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075950.163730:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075950.163735:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2939 00000100:00080000:1.0:1769075950.163738:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075950.163743:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075950.163748:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075950.163768:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617202560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075950.163769:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617202816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075950.163799:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075950.163802:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075950.163804:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075950.163810:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2939 00000100:00080000:1.0:1769075950.163816:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075950.163820:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075950.163826:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769075950.163840:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617202944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075950.163853:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075950.163857:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075950.163859:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075950.163863:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2939 00000100:00080000:1.0:1769075950.163879:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075950.163884:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075950.163892:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769075950.163894:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075950.163911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617203072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075950.163978:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075950/real 1769075950] req@ffff9b0343dec700 x1855007617202560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075950.164010:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617202560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075950.164025:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075950.164031:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075950.164036:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617202560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075950.164047:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075950.164080:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075950.164097:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075950/real 1769075950] req@ffff9b0343dec000 x1855007617202688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075950.164112:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617202688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075950.164123:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075950.164127:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075950.164131:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075950.164161:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075950.164180:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075950.164192:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075950/real 1769075950] req@ffff9b0343dedf80 x1855007617202816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075950.164206:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617202816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075950.164216:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075950.164219:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075950.164223:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075950.164230:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075950.164310:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075950.164325:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075950/real 1769075950] req@ffff9b0343ded180 x1855007617202944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075950.164373:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617202944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075950.164385:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075950.164389:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075950.164394:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617202944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075950.164402:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075950.164429:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075950.164443:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075950/real 1769075950] req@ffff9b0343dec380 x1855007617203072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075950.164458:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617203072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075950.164468:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075950.164472:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075950.164475:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075950.675399:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075951.059407:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075951.059452:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075951.059453:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075951.699379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075952.083392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075952.083399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075952.083526:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075952.723395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075953.107415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075953.107419:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075953.107525:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075953.747367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075954.131359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075954.131429:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075954.131481:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769075954.771411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075955.155374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075955.155413:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075955.155569:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769075955.283487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075955.283506:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769075955.283516:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075955.283531:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2944 00000100:00080000:1.0:1769075955.283548:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075955.283558:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769075955.283616:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617203200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075955.283653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075955.283660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769075955.283664:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075955.283681:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2944 00000100:00080000:1.0:1769075955.283686:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075955.283694:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075955.283708:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075955.283710:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617203200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075955.283733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617203328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075955.283750:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075955.283755:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769075955.283759:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075955.283773:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2944 00000100:00080000:1.0:1769075955.283800:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075955.283814:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075955.283823:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769075955.283832:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769075955.283854:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617203456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075955.283871:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769075955.283886:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769075955.283889:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075955.283897:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2944 00000100:00080000:1.0:1769075955.283906:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075955.283915:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769075955.283927:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075955.283931:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075955/real 1769075955] req@ffff9b0343dee300 x1855007617203200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769075955.283953:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617203584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769075955.283978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769075955.283981:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617203200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769075955.283986:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769075955.283993:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769075955.284010:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2944 00000100:00080000:0.0:1769075955.284018:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769075955.284019:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769075955.284033:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:0.0:1769075955.284037:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000008:00100000:1.0:1769075955.284042:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075955.284048:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617203200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769075955.284063:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617203712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075955.284088:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075955.284131:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075955.284147:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075955/real 1769075955] req@ffff9b0343def100 x1855007617203328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075955.284163:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617203328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075955.284175:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075955.284180:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075955.284185:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075955.284192:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075955.284214:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075955.284226:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075955/real 1769075955] req@ffff9b0343ded880 x1855007617203456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075955.284327:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617203456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075955.284341:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075955.284355:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075955.284362:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075955.284375:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075955.284403:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075955.284419:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075955/real 1769075955] req@ffff9b0343def800 x1855007617203584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075955.284435:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617203584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075955.284446:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075955.284451:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075955.284455:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075955.284462:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075955.284486:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075955.284503:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075955/real 1769075955] req@ffff9b0343deea00 x1855007617203712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075955.284523:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617203712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075955.284534:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075955.284540:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075955.284546:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075955.795454:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075956.179397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075956.179401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075956.179441:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075956.819340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075957.203382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075957.203403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075957.203529:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075957.843356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075958.227362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075958.227389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075958.227485:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075958.867417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075959.251360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075959.251368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075959.251449:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075959.891419:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075960.275309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075960.275309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075960.275376:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075960.403312:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075960.403321:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075960.403325:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075960.403334:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2949 00000100:00080000:3.0:1769075960.403342:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075960.403348:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075960.403382:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617203840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075960.403405:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075960.403408:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075960.403410:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075960.403413:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2949 00000100:00080000:3.0:1769075960.403415:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075960.403418:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075960.403425:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075960.403435:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617203968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075960.403440:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617203840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075960.403452:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075960.403456:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075960.403457:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075960.403460:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2949 00000100:00080000:3.0:1769075960.403463:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075960.403466:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075960.403469:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075960.403477:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617204096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075960.403484:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075960.403487:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075960.403488:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075960.403490:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2949 00000100:00080000:3.0:1769075960.403493:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075960.403495:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075960.403498:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769075960.403503:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075960.403507:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617204224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075960.403515:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075960.403519:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075960.403522:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075960.403526:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2949 00000100:00080000:3.0:1769075960.403530:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075960.403534:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075960.403540:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075960/real 1769075960] req@ffff9b0352a19500 x1855007617203840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769075960.403540:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075960.403561:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617204352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075960.403562:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617203840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075960.403584:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075960.403592:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075960.403599:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617203840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075960.403621:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075960.403636:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075960.403644:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075960/real 1769075960] req@ffff9b0352a1b800 x1855007617203968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075960.403652:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617203968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075960.403658:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075960.403660:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075960.403663:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617203968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075960.403669:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075960.403687:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075960.403698:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075960/real 1769075960] req@ffff9b0352a18380 x1855007617204096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075960.403719:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617204096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075960.403728:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075960.403731:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075960.403733:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075960.403738:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075960.403756:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075960.403767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075960/real 1769075960] req@ffff9b0352a19c00 x1855007617204224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075960.403780:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617204224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075960.403789:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075960.403792:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075960.403796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075960.403802:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075960.403818:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075960.403825:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075960/real 1769075960] req@ffff9b0352a18000 x1855007617204352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075960.403833:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617204352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075960.403842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075960.403844:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075960.403847:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075960.915395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075961.299348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769075961.299419:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769075961.299457:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075961.939371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075962.323415:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075962.323602:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075962.323723:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075962.963328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075963.347353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769075963.347355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075963.347493:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075963.987395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075964.371425:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075964.371463:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075964.371505:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075965.011339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075965.395491:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075965.395720:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075965.396579:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075965.523430:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075965.523445:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075965.523451:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075965.523465:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2954 00000100:00080000:0.0:1769075965.523476:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075965.523485:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075965.523534:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617204480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075965.523583:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075965.523588:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075965.523590:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075965.523596:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2954 00000100:00080000:0.0:1769075965.523600:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075965.523606:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075965.523619:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075965.523641:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617204608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075965.523643:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617204480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075965.523671:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075965.523676:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075965.523679:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075965.523684:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2954 00000100:00080000:0.0:1769075965.523689:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075965.523693:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075965.523711:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075965.523732:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617204736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075965.523752:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769075965.523758:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769075965.523768:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075965.523776:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075965.523788:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2954 00000100:00080000:0.0:1769075965.523807:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075965.523817:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075965.523829:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075965.523833:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075965/real 1769075965] req@ffff9b0365a38a80 x1855007617204480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075965.523857:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617204864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769075965.523868:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617204480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075965.523894:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075965.523901:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075965.523902:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075965.523910:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075965.523916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:0.0:1769075965.523923:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2954 00000100:00100000:3.0:1769075965.523931:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617204480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075965.523934:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075965.523950:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075965.523957:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075965.523976:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075965.523978:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617204992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769075965.524033:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075965.524052:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075965/real 1769075965] req@ffff9b0365a39c00 x1855007617204608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075965.524069:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617204608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075965.524134:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075965.524139:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075965.524145:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075965.524153:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769075965.524178:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075965.524191:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075965/real 1769075965] req@ffff9b0365a38e00 x1855007617204736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075965.524207:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617204736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075965.524217:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075965.524220:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075965.524225:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769075965.524263:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075965.524287:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075965.524300:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075965/real 1769075965] req@ffff9b0365a3b480 x1855007617204864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075965.524315:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617204864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075965.524325:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075965.524329:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075965.524333:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769075965.524353:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769075965.524372:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769075965.524383:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075965/real 1769075965] req@ffff9b0365a3aa00 x1855007617204992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075965.524398:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617204992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075965.524408:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769075965.524411:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769075965.524415:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617204992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075966.035366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075966.419453:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769075966.419581:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075966.419777:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075967.059354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769075967.443362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075967.443448:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769075967.443556:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075968.083367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075968.467395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075968.467510:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075968.467597:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769075969.107382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075969.491358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769075969.491396:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075969.491473:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075970.131352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075970.515338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075970.515464:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769075970.515478:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769075970.643454:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075970.643464:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075970.643469:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075970.643479:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2959 00000100:00080000:3.0:1769075970.643491:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075970.643502:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075970.643539:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617205120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075970.643572:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075970.643577:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075970.643579:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075970.643582:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2959 00000100:00080000:3.0:1769075970.643585:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075970.643589:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075970.643598:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075970.643607:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617205248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075970.643617:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075970.643620:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075970.643622:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075970.643625:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2959 00000100:00080000:3.0:1769075970.643628:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075970.643631:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075970.643635:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075970.643646:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617205120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075970.643646:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617205376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075970.643664:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075970.643666:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075970.643668:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075970.643672:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2959 00000100:00080000:3.0:1769075970.643675:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075970.643678:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075970.643682:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075970.643693:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617205504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075970.643700:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075970.643704:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075970.643706:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075970.643711:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2959 00000100:00080000:3.0:1769075970.643716:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769075970.643721:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075970.643726:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075970.643736:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075970.643756:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617205632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075970.643768:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075970/real 1769075970] req@ffff9b0352a1a680 x1855007617205120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075970.643788:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617205120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075970.643798:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075970.643803:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075970.643807:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617205120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075970.643814:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617205248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075970.643831:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075970.643840:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075970/real 1769075970] req@ffff9b0352a1b100 x1855007617205248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075970.643850:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617205248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075970.643863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075970.643865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075970.643868:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617205248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075970.643889:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617205376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075970.643901:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075970.643909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075970/real 1769075970] req@ffff9b0352a1ad80 x1855007617205376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075970.643917:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617205376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075970.643923:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075970.643926:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075970.643928:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617205376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075970.643932:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617205504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075970.643944:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075970.643951:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075970/real 1769075970] req@ffff9b0352a1b480 x1855007617205504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075970.643960:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617205504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075970.643966:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075970.643968:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075970.643971:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617205504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075970.643976:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617205632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075970.643987:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075970.644010:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075970/real 1769075970] req@ffff9b0352a18a80 x1855007617205632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075970.644022:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617205632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769075995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075970.644046:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075970.644049:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075970.644052:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617205632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075971.155394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769075971.539388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075971.539426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075971.539570:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075972.179393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075972.563375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075972.563432:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075972.563508:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075973.203384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075973.587382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075973.587382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075973.587474:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075974.227362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075974.611314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075974.611318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075974.611357:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075975.251383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075975.635371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075975.635381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075975.635465:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075975.763374:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075975.763387:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075975.763394:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075975.763409:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2964 00000100:00080000:3.0:1769075975.763424:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075975.763436:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075975.763492:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617205760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075975.763538:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075975.763544:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075975.763547:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075975.763552:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2964 00000100:00080000:3.0:1769075975.763558:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075975.763563:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075975.763576:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075975.763593:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617205888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075975.763606:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075975.763611:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075975.763613:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075975.763617:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2964 00000100:00080000:3.0:1769075975.763621:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075975.763625:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075975.763630:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075975.763642:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617206016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075975.763654:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075975.763658:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075975.763660:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075975.763666:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2964 00000100:00080000:3.0:1769075975.763669:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075975.763674:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075975.763680:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075975.763698:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617205760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769075975.763698:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617206144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075975.763729:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075975.763732:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075975.763735:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075975.763739:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2964 00000100:00080000:3.0:1769075975.763743:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075975.763748:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075975.763754:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075975.763770:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617206272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769075975.763817:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075975.763868:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075975/real 1769075975] req@ffff9b0352a18700 x1855007617205760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075975.763892:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617205760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075975.763905:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075975.763960:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075975.763969:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617205760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075975.763984:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617205888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075975.764036:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075975.764061:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075975/real 1769075975] req@ffff9b0352a1aa00 x1855007617205888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075975.764086:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617205888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075975.764097:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075975.764101:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075975.764106:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617205888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075975.764113:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075975.764141:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075975.764153:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075975/real 1769075975] req@ffff9b0352a19f80 x1855007617206016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075975.764168:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617206016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075975.764178:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075975.764182:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075975.764185:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075975.764216:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075975.764283:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075975.764298:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075975/real 1769075975] req@ffff9b0352a1a300 x1855007617206144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075975.764323:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617206144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075975.764333:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075975.764337:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075975.764341:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075975.764348:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075975.764370:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075975.764380:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075975/real 1769075975] req@ffff9b0352a18e00 x1855007617206272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075975.764394:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617206272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075975.764404:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075975.764407:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075975.764411:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075976.275439:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075976.659319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075976.659322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075976.659388:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075977.299342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075977.683317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075977.683319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075977.683365:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075978.323360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075978.707358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769075978.707461:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075978.707538:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075979.347395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769075979.731352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075979.731362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075979.731487:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075980.371356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075980.755366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075980.755443:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075980.755515:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075980.883337:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075980.883347:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075980.883352:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075980.883362:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2970 00000100:00080000:3.0:1769075980.883372:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075980.883378:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075980.883413:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617206400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075980.883439:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075980.883443:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075980.883444:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075980.883448:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2970 00000100:00080000:3.0:1769075980.883450:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075980.883453:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075980.883462:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075980.883471:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617206528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075980.883479:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075980.883482:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769075980.883484:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075980.883487:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2970 00000100:00080000:3.0:1769075980.883489:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075980.883498:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075980.883504:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075980.883512:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617206656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075980.883523:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075980.883526:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075980.883528:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075980.883531:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2970 00000100:00080000:3.0:1769075980.883533:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075980.883535:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075980.883539:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075980.883549:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617206784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075980.883562:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617206400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075980.883562:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075980.883578:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075980.883590:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075980.883596:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2970 00000100:00080000:3.0:1769075980.883598:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075980.883601:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075980.883604:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075980.883618:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617206912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769075980.883795:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075980.883896:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075980.883937:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075980.883960:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769075981.395380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075981.779431:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075981.779459:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075981.779522:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075982.419306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075982.803363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075982.803373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075982.803383:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075983.443406:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075983.827396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075983.827396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075983.827414:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769075983.955789:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769075983.955880:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769075983.955927:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075983.955977:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075983.955999:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075983.956016:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769075983.956043:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769075983.956053:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075980/real 1769075983] req@ffff9b0352a18e00 x1855007617206400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075983.956107:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617206400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075983.956127:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075983.956137:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075983.956146:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617206400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075983.956159:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075980/real 1769075983] req@ffff9b0352a1a300 x1855007617206528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075983.956187:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617206528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075983.956201:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075983.956209:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075983.956215:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075983.956225:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075980/real 1769075983] req@ffff9b0352a19f80 x1855007617206656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075983.956280:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617206656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075983.956294:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075983.956302:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075983.956311:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075983.956353:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075980/real 1769075983] req@ffff9b0352a1aa00 x1855007617206784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075983.956373:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617206784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075983.956387:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075983.956394:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075983.956402:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075983.956413:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075980/real 1769075983] req@ffff9b0352a18700 x1855007617206912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075983.956433:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617206912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075983.956446:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075983.956451:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075983.956456:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617206912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075984.467435:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075984.851419:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075984.851420:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769075984.851576:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075985.491507:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075985.875352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075985.875352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075985.875428:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769075986.003423:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075986.003436:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769075986.003441:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075986.003451:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2975 00000100:00080000:3.0:1769075986.003460:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075986.003467:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769075986.003503:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617207040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075986.003536:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075986.003542:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769075986.003544:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075986.003550:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2975 00000100:00080000:3.0:1769075986.003556:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075986.003561:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075986.003575:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075986.003594:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617207168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075986.003610:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075986.003615:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1769075986.003618:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617207040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769075986.003627:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075986.003633:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2975 00000100:00080000:3.0:1769075986.003638:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075986.003644:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075986.003650:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769075986.003664:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617207296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075986.003676:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075986.003681:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769075986.003683:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769075986.003688:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2975 00000100:00080000:3.0:1769075986.003692:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769075986.003693:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769075986.003697:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075986.003704:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769075986.003717:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617207424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769075986.003729:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769075986.003734:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769075986.003759:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769075986.003763:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075986/real 1769075986] req@ffff9b0352a18a80 x1855007617207040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769075986.003765:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2975 00000100:00080000:3.0:1769075986.003770:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769075986.003776:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769075986.003783:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075986.003790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617207040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769075986.003797:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617207552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075986.003816:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075986.003823:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075986.003829:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617207040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075986.003861:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075986.003893:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075986.003908:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075986/real 1769075986] req@ffff9b0352a1b480 x1855007617207168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075986.003921:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617207168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075986.003933:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075986.003938:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075986.003943:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075986.003952:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769075986.003992:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075986.004015:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075986/real 1769075986] req@ffff9b0352a1ad80 x1855007617207296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075986.004035:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617207296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075986.004047:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075986.004053:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075986.004057:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769075986.004074:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075986.004105:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075986.004122:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075986/real 1769075986] req@ffff9b0352a1b100 x1855007617207424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075986.004140:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617207424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075986.004152:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075986.004157:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075986.004162:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769075986.004170:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769075986.004200:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075986.004216:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075986/real 1769075986] req@ffff9b0352a1a680 x1855007617207552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769075986.004297:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617207552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075986.004310:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769075986.004315:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769075986.004320:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075986.515309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075986.899393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769075986.899487:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075986.899487:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769075987.539404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075987.923374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075987.923374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075987.923470:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075988.563477:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075988.947366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075988.947367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075988.947465:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075989.587429:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075989.971402:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075989.971454:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769075989.971468:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075990.611306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075990.995351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075990.995355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075990.995417:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075991.123426:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075991.123435:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075991.123440:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075991.123450:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2980 00000100:00080000:0.0:1769075991.123460:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075991.123467:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075991.123525:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617207680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075991.123557:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075991.123563:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075991.123565:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075991.123571:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2980 00000100:00080000:0.0:1769075991.123577:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075991.123597:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075991.123607:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769075991.123623:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617207680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769075991.123631:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617207808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075991.123671:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075991.123677:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075991.123681:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075991.123688:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2980 00000100:00080000:0.0:1769075991.123695:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075991.123702:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075991.123711:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769075991.123713:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075991.123735:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617207936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075991.123758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075991/real 1769075991] req@ffff9b0365a39500 x1855007617207680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769075991.123760:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075991.123771:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075991.123776:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075991.123785:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2980 00000100:00080000:0.0:1769075991.123791:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075991.123797:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075991.123805:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769075991.123807:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617207680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075991.123823:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769075991.123829:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617208064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769075991.123849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075991.123858:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617207680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075991.123874:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075991.123880:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075991.123883:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075991.123889:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2980 00000100:00100000:2.0:1769075991.123895:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207808:192.168.204.160@tcp:38:kworker.0 00000100:00080000:0.0:1769075991.123897:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075991.123907:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075991.123915:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075991.123927:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075991.123935:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617208192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075991.123945:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075991/real 1769075991] req@ffff9b0365a3bb80 x1855007617207808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075991.123972:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617207808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075991.123987:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075991.123993:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075991.124001:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075991.124013:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075991.124044:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075991.124057:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075991/real 1769075991] req@ffff9b0365a3b100 x1855007617207936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075991.124067:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617207936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075991.124074:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075991.124077:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075991.124080:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617207936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075991.124086:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075991.124100:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075991.124107:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075991/real 1769075991] req@ffff9b0365a3ad80 x1855007617208064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075991.124116:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617208064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075991.124122:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075991.124125:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075991.124127:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075991.124141:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075991.124154:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075991.124161:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075991/real 1769075991] req@ffff9b0365a39180 x1855007617208192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075991.124170:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617208192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075991.124177:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075991.124179:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075991.124181:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075991.635346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075992.019390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075992.019392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075992.019492:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075992.659383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075993.043378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075993.043385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075993.043557:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075993.683390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075994.067355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075994.067378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075994.067499:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075994.707380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075995.091344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075995.091357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075995.091530:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075995.731434:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075996.115313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075996.115335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075996.115419:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769075996.243363:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075996.243381:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769075996.243389:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075996.243405:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2985 00000100:00080000:0.0:1769075996.243423:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075996.243434:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769075996.243497:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617208320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075996.243555:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075996.243561:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769075996.243591:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075996.243600:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2985 00000100:00080000:0.0:1769075996.243606:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075996.243613:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075996.243627:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075996.243649:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617208448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075996.243669:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769075996.243672:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617208320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769075996.243675:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769075996.243681:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075996.243695:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2985 00000100:00080000:0.0:1769075996.243699:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075996.243704:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075996.243710:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769075996.243729:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617208576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075996.243741:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075996.243746:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769075996.243750:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075996.243755:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2985 00000100:00080000:0.0:1769075996.243762:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075996.243768:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075996.243775:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769075996.243785:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769075996.243789:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617208704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769075996.243806:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769075996.243811:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769075996.243815:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769075996.243820:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2985 00000100:00080000:0.0:1769075996.243824:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769075996.243829:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769075996.243837:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769075996.243856:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617208832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769075996.243874:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075996/real 1769075996] req@ffff9b0365a39180 x1855007617208320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075996.243911:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617208320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075996.243926:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075996.243933:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075996.243938:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617208320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769075996.243971:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075996.244003:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075996.244020:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075996/real 1769075996] req@ffff9b0365a3ad80 x1855007617208448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075996.244036:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617208448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075996.244048:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075996.244052:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075996.244057:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075996.244066:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769075996.244090:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075996.244104:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075996/real 1769075996] req@ffff9b0365a3b100 x1855007617208576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075996.244118:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617208576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075996.244129:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075996.244133:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075996.244138:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769075996.244146:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075996.244169:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075996.244182:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075996/real 1769075996] req@ffff9b0365a3bb80 x1855007617208704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075996.244197:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617208704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075996.244208:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075996.244211:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075996.244216:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075996.244224:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769075996.244293:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769075996.244307:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769075996/real 1769075996] req@ffff9b0365a39500 x1855007617208832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769075996.244322:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617208832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769075996.244333:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769075996.244338:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769075996.244342:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617208832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769075996.755342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769075997.139435:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769075997.139444:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769075997.139529:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075997.779337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075998.163321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075998.163329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075998.163408:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075998.803418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769075999.187325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769075999.187338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769075999.187390:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769075999.827401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076000.211325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076000.211325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076000.211383:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076000.851379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076001.235322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076001.235405:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076001.235408:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:0.0:1769076001.363380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076001.363398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076001.363404:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076001.363418:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2990 00000100:00080000:0.0:1769076001.363430:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076001.363439:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076001.363506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617208960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076001.363543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076001.363548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076001.363550:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076001.363555:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2990 00000100:00080000:0.0:1769076001.363560:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076001.363564:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076001.363574:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076001.363589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617209088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076001.363613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076001.363618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076001.363621:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076001.363625:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2990 00000100:00080000:0.0:1769076001.363628:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076001.363632:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076001.363637:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076001.363648:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617209216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076001.363657:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617208960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076001.363665:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076001.363670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076001.363672:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076001.363676:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2990 00000100:00080000:0.0:1769076001.363681:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076001.363685:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076001.363691:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076001.363705:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617209344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076001.363717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076001.363721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076001.363723:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076001.363737:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2990 00000100:00080000:0.0:1769076001.363743:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076001.363750:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076001.363754:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076001.363763:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076001.363777:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617209472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076001.363809:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076001/real 1769076001] req@ffff9b0365a3aa00 x1855007617208960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076001.363841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617208960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076001.363853:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076001.363858:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076001.363863:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617208960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076001.363872:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076001.363895:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076001.363913:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076001/real 1769076001] req@ffff9b0365a3b480 x1855007617209088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076001.363926:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617209088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076001.363935:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076001.363939:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076001.363942:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076001.363977:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076001.363994:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076001.364003:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076001/real 1769076001] req@ffff9b0365a38e00 x1855007617209216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076001.364015:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617209216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076001.364035:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076001.364038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076001.364042:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076001.364048:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076001.364063:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076001.364072:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076001/real 1769076001] req@ffff9b0365a39c00 x1855007617209344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076001.364085:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617209344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076001.364093:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076001.364096:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076001.364099:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076001.364105:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076001.364119:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076001.364128:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076001/real 1769076001] req@ffff9b0365a38a80 x1855007617209472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076001.364139:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617209472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076001.364147:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076001.364150:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076001.364153:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076001.875425:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076002.259398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076002.259404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076002.259531:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076002.899309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076003.283355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076003.283384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076003.283425:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076003.923344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076004.307355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076004.307369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076004.307516:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076004.947483:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076005.331353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076005.331395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076005.331501:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076005.971397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076006.355392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076006.355514:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076006.355551:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769076006.483526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076006.483541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076006.483548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076006.483561:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 2995 00000100:00080000:2.0:1769076006.483575:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076006.483584:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076006.483633:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617209600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076006.483687:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076006.483692:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076006.483695:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076006.483701:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2995 00000100:00080000:2.0:1769076006.483705:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076006.483709:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076006.483721:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076006.483740:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617209728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076006.483753:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076006.483758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076006.483760:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076006.483765:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2995 00000100:00080000:2.0:1769076006.483770:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076006.483774:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076006.483781:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076006.483791:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617209600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076006.483804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617209856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076006.483824:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076006.483838:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076006.483840:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076006.483849:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2995 00000100:00080000:2.0:1769076006.483855:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076006.483862:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076006.483869:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076006.483890:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617209984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076006.483903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076006.483907:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076006.483910:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076006.483914:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 2995 00000100:00080000:2.0:1769076006.483918:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076006.483922:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076006.483930:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076006.483944:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617210112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076006.483953:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076006.484042:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076006/real 1769076006] req@ffff9b03482c5180 x1855007617209600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076006.484072:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617209600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076006.484088:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076006.484095:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076006.484102:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617209600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076006.484113:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076006.484141:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076006.484156:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076006/real 1769076006] req@ffff9b03482c4700 x1855007617209728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076006.484172:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617209728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076006.484183:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076006.484188:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076006.484192:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076006.484287:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076006.484318:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076006.484334:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076006/real 1769076006] req@ffff9b03482c7100 x1855007617209856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076006.484349:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617209856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076006.484359:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076006.484363:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076006.484367:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076006.484376:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076006.484428:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076006.484451:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076006/real 1769076006] req@ffff9b03482c4000 x1855007617209984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076006.484477:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617209984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076006.484494:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076006.484499:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076006.484504:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617209984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076006.484513:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617210112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076006.484544:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076006.484558:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076006/real 1769076006] req@ffff9b03482c6a00 x1855007617210112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076006.484573:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617210112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076006.484584:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076006.484588:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076006.484591:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617210112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076006.995503:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769076007.379418:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076007.379500:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076007.379566:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076008.019346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076008.403316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076008.403352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076008.403413:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076009.043302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076009.427367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076009.427368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076009.427527:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076010.067353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076010.451406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076010.451422:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076010.451504:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076011.091427:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076011.475477:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076011.475496:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076011.475603:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076011.603437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076011.603457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076011.603465:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076011.603483:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3000 00000100:00080000:2.0:1769076011.603499:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076011.603511:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076011.603565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617210240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076011.603613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076011.603621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076011.603626:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076011.603634:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3000 00000100:00080000:2.0:1769076011.603642:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076011.603650:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076011.603664:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076011.603703:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617210368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076011.603729:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076011.603734:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617210240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076011.603739:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076011.603749:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076011.603763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3000 00000100:00080000:2.0:1769076011.603773:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076011.603781:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076011.603792:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076011.603820:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617210496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076011.603838:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076011.603856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076011.603864:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076011.603869:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076011.603881:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3000 00000100:00080000:2.0:1769076011.603900:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076011.603910:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076011/real 1769076011] req@ffff9b03482c5f80 x1855007617210240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076011.603913:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076011.603934:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076011.603948:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617210240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076011.603968:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769076011.603968:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617210624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076011.603982:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076011.603995:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617210240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076011.604021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076011.604030:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076011.604035:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076011.604047:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617210368:192.168.204.160@tcp:38:kworker.0 00000100:00080000:2.0:1769076011.604050:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3000 00000100:00080000:2.0:1769076011.604067:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076011.604078:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769076011.604089:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769076011.604091:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076011.604108:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076011/real 1769076011] req@ffff9b03482c7b80 x1855007617210368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076011.604141:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617210368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076011.604143:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617210752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076011.604166:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076011.604180:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076011.604192:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617210368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076011.604204:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617210496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076011.604227:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076011.604295:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076011/real 1769076011] req@ffff9b03482c4e00 x1855007617210496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076011.604343:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617210496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076011.604355:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076011.604371:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076011.604375:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617210496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076011.604384:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617210624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076011.604410:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076011.604424:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076011/real 1769076011] req@ffff9b03482c5c00 x1855007617210624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076011.604439:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617210624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076011.604449:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076011.604453:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076011.604457:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617210624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076011.604478:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617210752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076011.604496:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076011.604508:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076011/real 1769076011] req@ffff9b03482c6680 x1855007617210752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076011.604522:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617210752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076011.604532:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076011.604535:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076011.604538:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617210752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076012.115384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076012.499375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076012.499432:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076012.499542:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076013.139421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076013.523338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076013.523345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076013.523411:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076014.163445:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076014.547425:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076014.547470:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076014.547534:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076015.187466:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076015.571403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076015.571411:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076015.571588:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076016.211392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076016.595352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076016.595405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076016.595508:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076016.723415:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076016.723429:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076016.723434:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076016.723448:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3005 00000100:00080000:0.0:1769076016.723460:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076016.723469:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076016.723515:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617210880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076016.723573:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076016.723578:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076016.723580:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076016.723585:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3005 00000100:00080000:0.0:1769076016.723590:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076016.723595:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076016.723606:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076016.723623:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617211008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076016.723632:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617210880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076016.723659:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076016.723665:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076016.723667:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076016.723673:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3005 00000100:00080000:0.0:1769076016.723676:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076016.723682:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076016.723688:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076016.723703:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617211136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076016.723724:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076016.723728:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076016.723730:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076016.723735:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3005 00000100:00080000:0.0:1769076016.723751:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076016.723757:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769076016.723767:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769076016.723770:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076016.723790:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617211264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076016.723823:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076016.723829:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076016.723833:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076016.723858:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3005 00000100:00080000:0.0:1769076016.723867:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076016.723877:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076016/real 1769076016] req@ffff9b0365a38a80 x1855007617210880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076016.723878:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076016.723894:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076016.723916:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617210880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076016.723922:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617211392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076016.724006:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076016.724021:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076016.724032:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617210880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076016.724067:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076016.724104:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076016.724122:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076016/real 1769076016] req@ffff9b0365a39c00 x1855007617211008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076016.724139:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617211008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076016.724151:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076016.724155:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076016.724161:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076016.724169:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076016.724190:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076016.724201:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076016/real 1769076016] req@ffff9b0365a38e00 x1855007617211136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076016.724293:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617211136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076016.724302:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076016.724306:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076016.724310:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076016.724336:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076016.724366:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076016.724380:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076016/real 1769076016] req@ffff9b0365a3b480 x1855007617211264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076016.724395:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617211264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076016.724405:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076016.724409:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076016.724413:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076016.724420:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076016.724439:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076016.724451:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076016/real 1769076016] req@ffff9b0365a3aa00 x1855007617211392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076016.724465:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617211392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076016.724474:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076016.724478:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076016.724481:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076017.235377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076017.619365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076017.619367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076017.619581:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076018.259340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076018.643322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076018.643332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076018.643656:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076019.283318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076019.667353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076019.667363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076019.667508:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076020.307401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076020.691321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076020.691363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076020.691386:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076021.331361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076021.715333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076021.715399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076021.715530:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076021.843410:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076021.843423:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076021.843429:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076021.843442:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3011 00000100:00080000:2.0:1769076021.843454:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076021.843463:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076021.843511:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617211520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076021.843554:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076021.843560:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076021.843562:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076021.843567:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3011 00000100:00080000:2.0:1769076021.843573:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076021.843579:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076021.843593:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076021.843617:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617211648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076021.843620:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617211520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076021.843649:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076021.843653:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076021.843656:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076021.843662:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3011 00000100:00080000:2.0:1769076021.843667:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076021.843672:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076021.843678:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076021.843694:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617211776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076021.843706:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076021.843709:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076021.843712:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076021.843718:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3011 00000100:00080000:2.0:1769076021.843723:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076021.843744:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076021.843746:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076021.843763:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076021.843788:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617211904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076021.843809:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076021.843817:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076021.843823:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076021.843828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076021/real 1769076021] req@ffff9b03482c5880 x1855007617211520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076021.843838:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3011 00000100:00080000:2.0:1769076021.843853:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076021.843868:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076021.843877:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076021.843883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617211520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076021.843902:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076021.843911:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076021.843917:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617211520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076021.843919:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617212032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076021.843965:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076021.844003:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076021.844036:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076021/real 1769076021] req@ffff9b03482c6680 x1855007617211648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076021.844053:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617211648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076021.844064:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076021.844069:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076021.844073:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076021.844081:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076021.844103:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076021.844125:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076021/real 1769076021] req@ffff9b03482c5c00 x1855007617211776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076021.844170:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617211776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076021.844189:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076021.844193:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076021.844198:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076021.844206:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076021.844287:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076021.844312:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076021/real 1769076021] req@ffff9b03482c4e00 x1855007617211904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076021.844339:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617211904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076021.844350:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076021.844355:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076021.844359:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617211904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076021.844367:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076021.844395:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076021.844409:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076021/real 1769076021] req@ffff9b03482c7b80 x1855007617212032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076021.844423:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617212032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076021.844433:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076021.844436:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076021.844440:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076022.355327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076022.739373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076022.739424:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076022.739445:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076023.379355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076023.763356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076023.763371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076023.763481:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076024.403314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076024.787376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076024.787381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076024.787538:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076025.427341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076025.811326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076025.811508:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076025.811516:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076026.451330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076026.835377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076026.835481:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076026.835518:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076026.963378:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076026.963390:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076026.963396:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076026.963409:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3016 00000100:00080000:2.0:1769076026.963422:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076026.963433:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076026.963480:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617212160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076026.963520:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076026.963526:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076026.963529:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076026.963535:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3016 00000100:00080000:2.0:1769076026.963540:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076026.963546:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076026.963559:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076026.963590:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617212160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076026.963591:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617212288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076026.963620:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076026.963625:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076026.963628:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076026.963635:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3016 00000100:00080000:2.0:1769076026.963643:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076026.963652:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076026.963662:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076026.963688:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617212416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076026.963708:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076026.963716:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076026.963726:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076026.963732:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076026.963743:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3016 00000100:00080000:2.0:1769076026.963753:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076026.963762:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076026.963770:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076026/real 1769076026] req@ffff9b03482c5f80 x1855007617212160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076026.963773:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076026.963803:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617212544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076026.963812:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617212160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076026.963834:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076026.963848:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076026.963852:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076026.963853:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076026.963871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:2.0:1769076026.963872:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3016 00000100:00080000:2.0:1769076026.963884:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076026.963887:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617212160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076026.963895:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076026.963904:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076026.963918:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617212672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076026.963930:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076026.963974:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076026.963997:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076026/real 1769076026] req@ffff9b03482c6a00 x1855007617212288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076026.964013:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617212288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076026.964024:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076026.964028:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076026.964033:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076026.964040:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076026.964094:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076026.964110:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076026/real 1769076026] req@ffff9b03482c4000 x1855007617212416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076026.964126:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617212416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076026.964136:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076026.964140:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076026.964144:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076026.964152:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076026.964172:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076026.964184:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076026/real 1769076026] req@ffff9b03482c7100 x1855007617212544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076026.964199:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617212544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076026.964208:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076026.964213:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076026.964218:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076026.964285:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076026.964313:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076026.964330:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076026/real 1769076026] req@ffff9b03482c4700 x1855007617212672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076026.964347:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617212672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076026.964360:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076026.964365:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076026.964370:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076027.475357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076027.859448:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076027.859457:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076027.859787:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076028.499436:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076028.883796:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076028.883886:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076028.883981:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076029.523367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076029.907634:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076029.907879:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076029.908166:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076030.547347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076030.931598:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076030.931700:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076030.931733:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076031.571360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076031.955367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076031.955390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076031.955785:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076032.083350:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076032.083364:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076032.083371:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076032.083387:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3021 00000100:00080000:2.0:1769076032.083407:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076032.083422:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076032.083496:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617212800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076032.083545:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076032.083552:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076032.083570:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076032.083576:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3021 00000100:00080000:2.0:1769076032.083583:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076032.083591:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076032.083611:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076032.083640:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617212928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076032.083659:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617212800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076032.083670:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076032.083681:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076032.083686:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076032.083697:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3021 00000100:00080000:2.0:1769076032.083707:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076032.083715:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076032.083724:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076032.083745:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617213056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076032.083758:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076032.083763:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076032.083766:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076032.083772:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3021 00000100:00080000:2.0:1769076032.083778:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076032.083782:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076032.083789:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769076032.083791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076032.083812:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617213184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076032.083827:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076032.083833:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076032.083835:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076032.083841:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3021 00000100:00080000:2.0:1769076032.083846:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076032.083851:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076032.083858:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076032.083867:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076032/real 1769076032] req@ffff9b03482c5180 x1855007617212800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076032.083877:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617213312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076032.083926:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617212800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076032.083944:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076032.083951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076032.083957:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617212800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076032.084041:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076032.084076:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076032.084093:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076032/real 1769076032] req@ffff9b03482c6d80 x1855007617212928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076032.084110:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617212928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076032.084121:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076032.084127:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076032.084131:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617212928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076032.084139:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076032.084159:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076032.084170:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076032/real 1769076032] req@ffff9b03482c6300 x1855007617213056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076032.084185:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617213056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076032.084195:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076032.084199:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076032.084202:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076032.084210:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076032.084228:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076032.084289:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076032/real 1769076032] req@ffff9b03482c4a80 x1855007617213184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076032.084305:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617213184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076032.084316:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076032.084320:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076032.084324:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076032.084331:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076032.084355:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076032.084367:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076032/real 1769076032] req@ffff9b03482c4380 x1855007617213312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076032.084381:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617213312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076032.084392:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076032.084396:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076032.084400:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076032.596344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076032.979561:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076032.979666:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076032.979679:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076033.619339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076034.003412:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076034.003412:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076034.003595:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076034.643295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076035.027340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076035.027377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076035.027490:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076035.667290:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076036.051395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076036.051414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076036.051515:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076036.691348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076037.075400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076037.075409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076037.075501:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076037.203395:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076037.203411:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076037.203418:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076037.203432:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3026 00000100:00080000:3.0:1769076037.203446:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076037.203454:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076037.203493:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617213440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076037.203524:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076037.203530:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076037.203533:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076037.203539:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3026 00000100:00080000:3.0:1769076037.203546:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076037.203551:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076037.203564:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076037.203582:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617213568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076037.203584:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617213440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076037.203598:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076037.203603:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076037.203606:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076037.203612:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3026 00000100:00080000:3.0:1769076037.203617:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076037.203622:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076037.203628:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076037.203640:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617213696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076037.203652:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076037.203656:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1769076037.203659:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076037.203659:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076037.203664:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3026 00000100:00080000:3.0:1769076037.203668:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076037.203673:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076037.203678:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076037.203691:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617213824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076037.203703:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076037.203708:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076037.203711:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076037.203717:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3026 00000100:00100000:0.0:1769076037.203722:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076037/real 1769076037] req@ffff9b0352a18000 x1855007617213440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076037.203722:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076037.203727:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076037.203733:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076037.203748:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617213440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076037.203748:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617213952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076037.203765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076037.203770:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076037.203776:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617213440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076037.203808:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076037.203840:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076037.203854:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076037/real 1769076037] req@ffff9b0352a19c00 x1855007617213568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076037.203874:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617213568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076037.203904:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076037.203907:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076037.203910:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076037.203917:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076037.203936:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076037.203946:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076037/real 1769076037] req@ffff9b0352a18380 x1855007617213696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076037.203957:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617213696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076037.203965:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076037.203968:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076037.203971:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076037.203976:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076037.203990:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076037.203998:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076037/real 1769076037] req@ffff9b0352a1b800 x1855007617213824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076037.204008:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617213824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076037.204021:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076037.204024:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076037.204027:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076037.204032:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076037.204044:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076037.204052:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076037/real 1769076037] req@ffff9b0352a19500 x1855007617213952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076037.204062:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617213952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076037.204069:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076037.204072:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076037.204075:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617213952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076037.715316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769076038.099449:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076038.099459:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076038.099459:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076038.740308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076039.123357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076039.123408:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076039.123525:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076039.763347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076040.147362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076040.147422:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076040.147508:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076040.787356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076041.171379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076041.171435:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076041.171622:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076041.811397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076042.195345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076042.195363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076042.195434:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076042.323380:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076042.323393:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076042.323400:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076042.323431:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3031 00000100:00080000:3.0:1769076042.323442:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076042.323450:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076042.323491:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617214080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076042.323522:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076042.323546:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076042.323549:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076042.323554:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3031 00000100:00080000:3.0:1769076042.323559:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076042.323563:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076042.323574:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076042.323591:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617214208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076042.323620:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076042.323627:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076042.323632:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076042.323639:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617214080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076042.323648:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3031 00000100:00080000:3.0:1769076042.323659:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076042.323668:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076042.323680:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076042.323710:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617214336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076042.323735:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076042.323740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076042.323750:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076042.323756:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076042.323765:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3031 00000100:00080000:3.0:1769076042.323773:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076042.323779:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076042.323789:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076042.323825:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076042/real 1769076042] req@ffff9b0352a19500 x1855007617214080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076042.323858:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617214464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076042.323870:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076042.323872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617214080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076042.323877:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076042.323882:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076042.323895:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3031 00000100:00080000:0.0:1769076042.323900:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076042.323902:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076042.323910:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:0.0:1769076042.323913:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000008:00100000:3.0:1769076042.323917:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076042.323923:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617214080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076042.323935:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617214592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076042.323937:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076042.323977:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076042.323993:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076042/real 1769076042] req@ffff9b0352a1b800 x1855007617214208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076042.324008:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617214208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076042.324016:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076042.324020:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076042.324023:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076042.324065:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076042.324089:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076042.324101:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076042/real 1769076042] req@ffff9b0352a18380 x1855007617214336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076042.324115:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617214336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076042.324127:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076042.324132:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076042.324136:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076042.324145:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076042.324171:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076042.324213:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076042/real 1769076042] req@ffff9b0352a19c00 x1855007617214464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076042.324297:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617214464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076042.324311:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076042.324316:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076042.324320:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076042.324329:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076042.324359:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076042.324373:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076042/real 1769076042] req@ffff9b0352a18000 x1855007617214592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076042.324388:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617214592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076042.324398:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076042.324402:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076042.324406:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076042.835359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076043.219323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076043.219340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076043.219443:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076043.859334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076044.243326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076044.243357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076044.243517:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076044.883343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076045.267379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076045.267408:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076045.267530:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076045.907347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076046.291368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076046.291368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076046.291401:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076046.931379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076047.315331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076047.315445:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076047.315508:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076047.443459:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076047.443492:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076047.443499:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076047.443514:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3036 00000100:00080000:0.0:1769076047.443531:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076047.443544:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076047.443627:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617214720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076047.443686:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076047.443693:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076047.443696:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076047.443727:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3036 00000100:00080000:0.0:1769076047.443736:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076047.443746:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076047.443775:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076047.443793:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617214720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076047.443804:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617214848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076047.443829:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076047.443836:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076047.443839:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076047.443848:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3036 00000100:00080000:0.0:1769076047.443855:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076047.443875:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076047.443888:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076047.443913:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617214976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076047.443932:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076047.443940:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076047.443945:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076047.443955:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3036 00000100:00080000:0.0:1769076047.443965:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076047.443975:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076047.443986:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076047.444020:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076047.444038:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617215104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076047.444056:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076047.444060:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076047.444088:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076047.444095:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3036 00000100:00080000:0.0:1769076047.444104:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076047.444114:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076047.444119:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214976:192.168.204.160@tcp:38:kworker.0 00000008:00100000:0.0:1769076047.444127:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076047.444156:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617215232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076047.444173:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076047.444224:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076047.955331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076048.339371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076048.339375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076048.339501:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076048.979350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076049.363344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076049.363370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076049.363474:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076050.003371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076050.387347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076050.387395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076050.387508:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769076050.515570:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769076050.515621:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076050.515640:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076050.515746:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076050.515775:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076050.515790:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076050.515821:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769076050.515831:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076047/real 1769076050] req@ffff9b0365a3ad80 x1855007617215104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076050.515864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617215104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076050.515879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076050.515885:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076050.515891:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076050.515900:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076047/real 1769076050] req@ffff9b0365a3bb80 x1855007617214848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076050.515916:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617214848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076050.515927:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076050.515931:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076050.515935:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076050.515943:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076047/real 1769076050] req@ffff9b0365a3b100 x1855007617214976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076050.515972:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617214976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076050.515981:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076050.515985:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076050.515989:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617214976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076050.515997:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076047/real 1769076050] req@ffff9b0365a39500 x1855007617214720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076050.516011:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617214720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076050.516020:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076050.516025:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076050.516029:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617214720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076050.516067:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076047/real 1769076050] req@ffff9b0365a39180 x1855007617215232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076050.516082:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617215232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076050.516093:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076050.516097:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076050.516106:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076051.027347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076051.411344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076051.411391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076051.411410:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076052.051797:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076052.435385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076052.435389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076052.435601:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076052.563484:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076052.563501:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076052.563508:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076052.563521:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3041 00000100:00080000:3.0:1769076052.563533:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076052.563542:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076052.563584:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617215360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076052.563642:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076052.563649:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076052.563653:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076052.563678:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3041 00000100:00080000:3.0:1769076052.563686:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076052.563694:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076052.563709:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076052.563719:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617215360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076052.563741:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617215488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076052.563763:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076052.563769:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076052.563774:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076052.563783:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3041 00000100:00080000:3.0:1769076052.563805:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076052.563813:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076052.563824:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076052.563825:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076052.563854:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617215616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076052.563872:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076052.563878:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076052.563882:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076052.563894:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3041 00000100:00080000:3.0:1769076052.563901:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076052.563906:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076052/real 1769076052] req@ffff9b0352a1a680 x1855007617215360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076052.563909:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076052.563932:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076052.563960:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617215744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076052.563973:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617215360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076052.563983:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076052.563998:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076052.564007:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076052.564011:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076052.564019:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3041 00000100:00080000:2.0:1769076052.564028:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769076052.564032:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076052.564042:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617215360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076052.564045:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076052.564063:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076052.564086:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617215872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076052.564094:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076052.564157:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076052.564182:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076052/real 1769076052] req@ffff9b0352a1b100 x1855007617215488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076052.564205:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617215488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076052.564221:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076052.564229:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076052.564276:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076052.564297:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076052.564335:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076052.564357:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076052/real 1769076052] req@ffff9b0352a1ad80 x1855007617215616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076052.564379:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617215616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076052.564393:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076052.564398:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076052.564404:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076052.564415:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076052.564450:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076052.564470:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076052/real 1769076052] req@ffff9b0352a1b480 x1855007617215744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076052.564491:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617215744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076052.564504:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076052.564509:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076052.564514:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076052.564539:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076052.564571:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076052.564589:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076052/real 1769076052] req@ffff9b0352a18a80 x1855007617215872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076052.564610:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617215872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076052.564652:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076052.564658:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076052.564664:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617215872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076053.076276:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076053.459318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076053.459334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076053.459416:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076054.099373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076054.483350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076054.483436:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076054.483553:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076055.123524:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076055.507348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076055.507432:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076055.507506:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076056.147495:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076056.531394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076056.531406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076056.531520:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076057.171494:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076057.555350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076057.555452:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076057.555591:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076057.683429:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076057.683443:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076057.683450:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076057.683462:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3046 00000100:00080000:0.0:1769076057.683475:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076057.683486:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076057.683569:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617216000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076057.683609:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076057.683614:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076057.683616:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076057.683623:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3046 00000100:00080000:0.0:1769076057.683627:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076057.683633:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076057.683646:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076057.683664:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617216128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076057.683673:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617216000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076057.683695:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076057.683700:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076057.683702:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076057.683708:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3046 00000100:00080000:0.0:1769076057.683713:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076057.683717:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076057.683723:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076057.683741:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617216256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076057.683753:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076057.683758:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076057.683762:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076057.683767:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3046 00000100:00080000:0.0:1769076057.683773:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076057.683781:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076057.683787:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076057.683797:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076057.683814:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617216384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076057.683827:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076057.683831:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076057.683834:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076057.683839:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3046 00000100:00080000:0.0:1769076057.683847:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076057.683849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076057/real 1769076057] req@ffff9b0365a39180 x1855007617216000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076057.683856:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076057.683868:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076057.683888:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617216512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076057.683910:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617216000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076057.683936:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076057.683946:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076057.683954:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617216000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076057.683995:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076057.684030:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076057.684048:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076057/real 1769076057] req@ffff9b0365a39500 x1855007617216128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076057.684073:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617216128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076057.684084:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076057.684088:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076057.684093:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076057.684100:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076057.684119:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076057.684130:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076057/real 1769076057] req@ffff9b0365a3b100 x1855007617216256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076057.684144:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617216256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076057.684153:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076057.684156:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076057.684160:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076057.684167:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076057.684184:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076057.684195:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076057/real 1769076057] req@ffff9b0365a3bb80 x1855007617216384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076057.684208:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617216384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076057.684226:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076057.684262:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076057.684268:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076057.684275:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076057.684298:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076057.684311:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076057/real 1769076057] req@ffff9b0365a3ad80 x1855007617216512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076057.684325:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617216512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076057.684335:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076057.684339:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076057.684342:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076058.195436:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076058.579404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076058.579488:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076058.579494:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076059.219310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076059.603380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076059.603433:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076059.603489:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076060.243384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076060.627324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076060.627394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076060.627476:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076061.267330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076061.651352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076061.651352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076061.651447:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076062.291303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076062.675320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076062.675341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076062.675373:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076062.803424:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076062.803434:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076062.803439:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076062.803474:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3051 00000100:00080000:3.0:1769076062.803484:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076062.803493:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076062.803531:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617216640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076062.803577:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076062.803583:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076062.803586:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076062.803592:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3051 00000100:00080000:3.0:1769076062.803598:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076062.803603:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076062.803611:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617216640:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769076062.803615:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076062.803633:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617216768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076062.803647:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076062.803653:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076062.803656:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076062.803661:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3051 00000800:00000100:1.0:1769076062.803667:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076062.803667:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076062.803676:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076062.803684:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076062.803698:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617216896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076062.803704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076062/real 1769076062] req@ffff9b0352a18700 x1855007617216640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076062.803726:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617216640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076062.803734:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076062.803737:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076062.803740:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617216640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076062.803757:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076062.803772:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076062.803780:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076062/real 1769076062] req@ffff9b0352a1aa00 x1855007617216768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076062.803793:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617216768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076062.803796:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076062.803802:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076062.803804:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076062.803810:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769076062.803810:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076062.803816:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216768:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076062.803821:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3051 00000100:00100000:1.0:1769076062.803826:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216896:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076062.803829:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076062.803837:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769076062.803843:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769076062.803859:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076062.803865:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076062/real 1769076062] req@ffff9b0352a19f80 x1855007617216896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076062.803879:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617217024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076062.803884:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617216896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076062.803896:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076062.803899:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076062.803901:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617216896:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076062.804023:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076062.804028:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076062.804031:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076062.804037:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3051 00000100:00100000:1.0:1769076062.804051:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076062.804071:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076062.804082:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076062/real 1769076062] req@ffff9b0352a1a300 x1855007617217024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076062.804093:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617217024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076062.804100:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076062.804103:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076062.804107:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217024:192.168.204.160@tcp:8:kworker.0 00000100:00080000:3.0:1769076062.804112:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076062.804120:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076062.804129:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076062.804144:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617217152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076062.804188:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076062.804205:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076062.804214:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076062/real 1769076062] req@ffff9b0352a18e00 x1855007617217152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076062.804222:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617217152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076062.804228:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076062.804230:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076062.804266:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076063.315410:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076063.699359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076063.699378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076063.699517:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076064.339406:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076064.723389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076064.723389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076064.723525:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076065.363452:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076065.747360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076065.747451:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076065.747540:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076066.387350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076066.771375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076066.771407:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076066.771467:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076067.411352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076067.795313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076067.795325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076067.795382:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076067.923358:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076067.923398:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076067.923420:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076067.923428:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3057 00000100:00080000:3.0:1769076067.923438:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076067.923446:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076067.923509:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617217280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076067.923560:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076067.923566:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076067.923569:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076067.923574:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3057 00000100:00080000:3.0:1769076067.923603:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076067.923607:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076067.923616:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076067.923629:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617217280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076067.923630:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617217408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076067.923646:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076067.923649:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076067.923651:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076067.923656:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3057 00000100:00080000:3.0:1769076067.923658:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076067.923661:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076067.923665:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076067.923674:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617217536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076067.923681:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076067.923684:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076067.923686:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076067.923688:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3057 00000100:00080000:3.0:1769076067.923691:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076067.923694:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769076067.923699:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769076067.923700:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076067.923711:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617217664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076067.923718:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076067.923721:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076067.923722:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076067.923726:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3057 00000100:00080000:3.0:1769076067.923729:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076067.923732:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076067.923736:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076067.923737:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076067/real 1769076067] req@ffff9b0352a18e00 x1855007617217280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076067.923749:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617217792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076067.923772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617217280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076067.923788:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076067.923794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076067.923799:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617217280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076067.923825:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076067.923846:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076067.923856:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076067/real 1769076067] req@ffff9b0352a1a300 x1855007617217408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076067.923881:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617217408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076067.923888:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076067.923890:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076067.923892:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076067.923897:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076067.923909:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076067.923915:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076067/real 1769076067] req@ffff9b0352a19f80 x1855007617217536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076067.923923:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617217536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076067.923928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076067.923930:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076067.923932:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076067.923935:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076067.923957:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076067.923963:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076067/real 1769076067] req@ffff9b0352a1aa00 x1855007617217664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076067.923971:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617217664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076067.923977:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076067.923979:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076067.923981:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076067.923985:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076067.923994:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076067.924000:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076067/real 1769076067] req@ffff9b0352a18700 x1855007617217792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076067.924007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617217792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076067.924012:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076067.924014:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076067.924016:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617217792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076068.435412:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076068.819418:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076068.819470:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076068.819507:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076069.459456:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076069.843355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076069.843444:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076069.843598:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076070.483394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076070.867382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076070.867422:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076070.867572:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076071.507477:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076071.891444:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076071.891452:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076071.891494:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076072.531462:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076072.915306:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076072.915327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076072.915397:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076073.043405:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.043415:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076073.043419:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.043428:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3062 00000100:00080000:3.0:1769076073.043437:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.043444:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076073.043476:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617217920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.043501:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.043505:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076073.043506:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.043510:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3062 00000100:00080000:3.0:1769076073.043513:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.043515:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076073.043524:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076073.043533:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617218048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.043542:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.043545:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076073.043547:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.043551:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3062 00000100:00080000:3.0:1769076073.043553:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.043555:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076073.043559:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076073.043569:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617218176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.043577:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.043579:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769076073.043581:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617217920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076073.043581:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.043588:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3062 00000100:00080000:3.0:1769076073.043595:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.043598:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076073.043602:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076073.043609:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617218304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076073.043617:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076073.043620:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076073.043622:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076073.043625:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3062 00000100:00080000:3.0:1769076073.043627:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076073.043630:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076073.043633:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076073.043640:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617218432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076073.043657:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076073.043693:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076073/real 1769076073] req@ffff9b0352a18a80 x1855007617217920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076073.043708:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617217920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076073.043717:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076073.043721:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076073.043724:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617217920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076073.043729:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076073.043745:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076073.043753:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076073/real 1769076073] req@ffff9b0352a1b480 x1855007617218048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076073.043762:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617218048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076073.043768:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076073.043771:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076073.043773:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076073.043793:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076073.043805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076073.043811:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076073/real 1769076073] req@ffff9b0352a1ad80 x1855007617218176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076073.043820:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617218176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076073.043826:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076073.043828:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076073.043830:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076073.043834:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076073.043845:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076073.043851:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076073/real 1769076073] req@ffff9b0352a1b100 x1855007617218304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076073.043859:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617218304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076073.043864:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076073.043866:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076073.043868:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076073.043872:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076073.043881:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076073.043887:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076073/real 1769076073] req@ffff9b0352a1a680 x1855007617218432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076073.043900:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617218432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076073.043906:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076073.043908:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076073.043910:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076073.555391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076073.939407:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076073.939413:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076073.939459:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076074.579391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076074.963361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076074.963412:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076074.963528:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076075.603443:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076075.987342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076075.987361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076075.987610:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076076.627385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076077.011388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076077.011433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076077.011467:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076077.651400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076078.035333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076078.035351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076078.035514:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076078.163371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076078.163388:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076078.163395:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076078.163412:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3067 00000100:00080000:3.0:1769076078.163428:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076078.163440:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076078.163504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617218560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076078.163552:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076078.163557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076078.163560:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076078.163566:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3067 00000100:00080000:3.0:1769076078.163570:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076078.163577:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076078.163609:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076078.163639:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617218688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076078.163649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617218560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076078.163682:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076078.163691:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076078.163697:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076078.163706:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3067 00000100:00080000:3.0:1769076078.163712:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076078.163719:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076078.163727:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076078.163750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617218816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076078.163763:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076078.163767:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076078.163770:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076078.163776:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3067 00000100:00080000:3.0:1769076078.163784:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076078.163794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076078.163797:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076078.163812:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076078.163839:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617218944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076078.163861:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076078.163870:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076078/real 1769076078] req@ffff9b0352a18000 x1855007617218560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076078.163872:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076078.163881:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076078.163898:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3067 00000100:00080000:3.0:1769076078.163907:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076078.163916:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076078.163925:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076078.163937:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617218560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076078.163949:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617219072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076078.163990:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076078.164003:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076078.164012:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617218560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076078.164055:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076078.164093:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076078.164111:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076078/real 1769076078] req@ffff9b0352a19c00 x1855007617218688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076078.164129:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617218688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076078.164141:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076078.164146:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076078.164150:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076078.164158:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076078.164180:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076078.164227:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076078/real 1769076078] req@ffff9b0352a18380 x1855007617218816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076078.164291:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617218816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076078.164303:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076078.164307:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076078.164311:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076078.164321:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076078.164353:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076078.164369:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076078/real 1769076078] req@ffff9b0352a1b800 x1855007617218944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076078.164384:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617218944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076078.164395:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076078.164399:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076078.164403:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617218944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076078.164410:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076078.164428:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076078.164438:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076078/real 1769076078] req@ffff9b0352a19500 x1855007617219072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076078.164453:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617219072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076078.164463:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076078.164466:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076078.164470:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076078.676311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076079.059394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076079.059395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076079.059466:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076079.699348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076080.083362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076080.083363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076080.083531:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076080.723280:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076081.107359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076081.107374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076081.107445:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076081.747317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076082.131376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076082.131384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076082.131513:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076082.771325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076083.155406:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076083.155466:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076083.155584:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076083.283441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076083.283453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076083.283459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076083.283471:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3072 00000100:00080000:1.0:1769076083.283483:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076083.283492:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076083.283539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617219200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076083.283582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076083.283587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076083.283590:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076083.283595:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3072 00000100:00080000:1.0:1769076083.283600:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076083.283606:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076083.283617:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076083.283636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617219328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076083.283646:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617219200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076083.283660:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076083.283666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076083.283668:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076083.283674:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3072 00000100:00080000:1.0:1769076083.283679:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076083.283686:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076083.283693:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076083.283710:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617219456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076083.283722:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076083.283727:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076083.283730:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076083.283734:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3072 00000100:00080000:1.0:1769076083.283741:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076083.283756:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076083.283761:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076083.283771:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076083.283788:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617219584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076083.283800:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076083.283805:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076083.283808:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076083.283813:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3072 00000100:00100000:2.0:1769076083.283820:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076083/real 1769076083] req@ffff9b0343deed80 x1855007617219200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076083.283823:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076083.283836:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076083.283848:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076083.283874:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617219200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076083.283875:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617219712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076083.283936:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076083.283947:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076083.283953:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617219200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076083.283996:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076083.284023:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076083.284038:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076083/real 1769076083] req@ffff9b0343deca80 x1855007617219328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076083.284063:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617219328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076083.284074:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076083.284078:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076083.284082:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076083.284090:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076083.284138:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076083.284153:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076083/real 1769076083] req@ffff9b0343dece00 x1855007617219456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076083.284175:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617219456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076083.284185:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076083.284189:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076083.284193:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076083.284200:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076083.284221:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076083.284231:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076083/real 1769076083] req@ffff9b0343defb80 x1855007617219584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076083.284277:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617219584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076083.284296:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076083.284300:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076083.284304:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076083.284311:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076083.284333:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076083.284346:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076083/real 1769076083] req@ffff9b0343dee680 x1855007617219712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076083.284359:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617219712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076083.284369:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076083.284372:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076083.284376:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076083.795304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076084.179315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076084.179323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076084.179391:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076084.819396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076085.203348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076085.203372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076085.203485:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076085.843423:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076086.227382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076086.227456:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076086.227503:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076086.867438:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076087.251322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076087.251363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076087.251475:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076087.891376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076088.275382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076088.275386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076088.275559:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076088.403433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076088.403451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076088.403460:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076088.403482:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3077 00000100:00080000:3.0:1769076088.403504:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076088.403519:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076088.403596:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617219840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076088.403656:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076088.403665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076088.403670:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076088.403680:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3077 00000100:00080000:3.0:1769076088.403690:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076088.403701:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076088.403726:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076088.403727:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617219840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076088.403764:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617219968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076088.403788:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076088.403793:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076088.403795:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076088.403803:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3077 00000100:00080000:3.0:1769076088.403825:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076088.403838:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769076088.403840:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769076088.403859:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076088.403885:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617220096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076088.403904:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076088.403909:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076088.403912:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076088.403921:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076088/real 1769076088] req@ffff9b0352a19500 x1855007617219840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076088.403929:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3077 00000100:00080000:3.0:1769076088.403938:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076088.403950:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076088.403975:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076088.403994:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617219840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076088.403997:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617220224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076088.404058:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076088.404063:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076088.404069:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076088.404095:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3077 00000100:00080000:3.0:1769076088.404115:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076088.404132:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076088.404146:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769076088.404157:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769076088.404171:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617220352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076088.404181:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076088.404209:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617219840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076088.404294:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076088.404336:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076088.404354:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076088/real 1769076088] req@ffff9b0352a1b800 x1855007617219968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076088.404371:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617219968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076088.404383:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076088.404388:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076088.404392:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617219968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076088.404401:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076088.404421:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076088.404433:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076088/real 1769076088] req@ffff9b0352a18380 x1855007617220096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076088.404447:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617220096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076088.404457:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076088.404461:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076088.404464:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076088.404473:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076088.404491:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076088.404502:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076088/real 1769076088] req@ffff9b0352a19c00 x1855007617220224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076088.404516:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617220224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076088.404525:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076088.404529:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076088.404532:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076088.404540:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076088.404575:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076088.404588:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076088/real 1769076088] req@ffff9b0352a18000 x1855007617220352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076088.404602:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617220352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076088.404612:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076088.404616:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076088.404620:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076088.915387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076089.299369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076089.299381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076089.299402:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076089.939381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076090.323415:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076090.323505:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076090.323508:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076090.963382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076091.347397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076091.347405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076091.347462:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076091.987400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076092.371381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076092.371388:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076092.371389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076093.011363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076093.395342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076093.395449:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076093.395473:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:0.0:1769076093.523557:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076093.523573:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076093.523580:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076093.523595:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3082 00000100:00080000:0.0:1769076093.523613:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076093.523624:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076093.523687:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617220480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076093.523735:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076093.523741:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076093.523744:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076093.523750:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3082 00000100:00080000:0.0:1769076093.523755:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076093.523759:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076093.523772:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076093.523788:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617220608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076093.523810:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076093.523815:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076093.523818:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076093.523824:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3082 00000100:00080000:0.0:1769076093.523832:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076093.523835:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617220480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076093.523843:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076093.523856:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076093.523874:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617220736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076093.523888:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076093.523893:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076093.523895:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076093.523900:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3082 00000100:00080000:0.0:1769076093.523905:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076093.523909:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076093.523915:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076093.523928:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617220864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076093.523978:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076093.523995:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076093.523999:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769076093.524012:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076093.524013:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3082 00000100:00080000:0.0:1769076093.524029:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076093.524039:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076093.524048:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076093.524072:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617220992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076093.524097:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076093/real 1769076093] req@ffff9b0365a3aa00 x1855007617220480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076093.524128:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617220480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076093.524144:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076093.524150:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076093.524156:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617220480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076093.524167:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076093.524203:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076093.524218:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076093/real 1769076093] req@ffff9b0365a3b480 x1855007617220608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076093.524275:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617220608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076093.524288:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076093.524292:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076093.524296:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076093.524347:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076093.524375:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076093.524390:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076093/real 1769076093] req@ffff9b0365a38e00 x1855007617220736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076093.524428:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617220736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076093.524439:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076093.524443:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076093.524447:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076093.524455:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076093.524488:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076093.524500:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076093/real 1769076093] req@ffff9b0365a39c00 x1855007617220864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076093.524515:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617220864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076093.524525:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076093.524528:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076093.524532:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076093.524539:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076093.524557:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076093.524571:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076093/real 1769076093] req@ffff9b0365a38a80 x1855007617220992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076093.524585:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617220992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076093.524616:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076093.524621:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076093.524625:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617220992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076094.035387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076094.419365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076094.419394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076094.419475:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076095.059363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769076095.443509:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076095.443699:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076095.443776:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076096.083312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076096.467536:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076096.467610:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076096.467650:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076097.107390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769076097.491563:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076097.491684:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076097.492318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076098.131342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076098.515442:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076098.515803:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076098.516361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:2.0:1769076098.643402:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076098.643416:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076098.643421:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076098.643436:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3087 00000100:00080000:2.0:1769076098.643451:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076098.643463:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076098.643512:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617221120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076098.643563:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076098.643569:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076098.643571:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076098.643578:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3087 00000100:00080000:2.0:1769076098.643585:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076098.643591:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076098.643604:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076098.643625:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617221120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076098.643633:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617221248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076098.643656:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076098.643661:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076098.643665:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076098.643671:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3087 00000100:00080000:2.0:1769076098.643678:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076098.643682:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076098.643690:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076098.643707:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617221376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076098.643721:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769076098.643733:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076098.643734:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076098.643740:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076098.643749:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3087 00000100:00080000:2.0:1769076098.643757:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076098.643762:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076098.643772:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076098.643790:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617221504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076098.643806:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076098.643810:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076098.643813:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076098.643819:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3087 00000100:00080000:2.0:1769076098.643826:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076098.643831:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076098.643837:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076098.643874:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617221632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076098.643978:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076098/real 1769076098] req@ffff9b03482c5500 x1855007617221120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076098.644008:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617221120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076098.644022:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076098.644028:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076098.644034:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617221120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076098.644066:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617221248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076098.644102:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076098.644332:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076098/real 1769076098] req@ffff9b03482c4380 x1855007617221248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076098.644352:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617221248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076098.644363:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076098.644368:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076098.644372:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617221248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076098.644452:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617221376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076098.644491:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076098.644522:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076098/real 1769076098] req@ffff9b03482c4a80 x1855007617221376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076098.644540:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617221376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076098.644553:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076098.644557:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076098.644562:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617221376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076098.644570:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617221504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076098.644590:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076098.644601:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076098/real 1769076098] req@ffff9b03482c6300 x1855007617221504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076098.644624:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617221504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076098.644634:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076098.644638:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076098.644642:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617221504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076098.644648:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617221632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076098.644665:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076098.644676:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076098/real 1769076098] req@ffff9b03482c6d80 x1855007617221632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076098.644689:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617221632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076098.644699:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076098.644702:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076098.644706:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617221632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076099.155400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076099.539472:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076099.539625:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076099.539825:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076100.179367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076100.563397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076100.563397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076100.563461:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076101.203293:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076101.587378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076101.587402:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076101.587519:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076102.227377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076102.611305:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076102.611305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076102.611360:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076103.251323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076103.635354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076103.635356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076103.635431:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076103.763422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076103.763437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076103.763443:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076103.763457:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3092 00000100:00080000:1.0:1769076103.763469:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076103.763478:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076103.763524:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617221760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076103.763562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076103.763568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076103.763570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076103.763576:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3092 00000100:00080000:1.0:1769076103.763594:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076103.763603:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076103.763621:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076103.763647:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617221760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076103.763656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617221888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076103.763687:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076103.763695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076103.763701:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076103.763711:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3092 00000100:00080000:1.0:1769076103.763721:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076103.763729:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076103.763738:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076103.763801:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076103.763804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617222016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076103.763820:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076103.763825:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076103.763827:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076103.763832:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3092 00000100:00080000:1.0:1769076103.763837:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076103.763843:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076103.763851:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076103.763859:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076103/real 1769076103] req@ffff9b0343ded500 x1855007617221760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076103.763869:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617222144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076103.763889:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076103.763894:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1769076103.763896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617221760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076103.763898:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076103.763912:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3092 00000100:00080000:1.0:1769076103.763929:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076103.763930:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076103.763936:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076103.763945:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1769076103.763945:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076103.763959:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617221760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076103.763965:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617222272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076103.764007:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617221888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076103.764051:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076103.764068:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076103/real 1769076103] req@ffff9b0343def480 x1855007617221888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076103.764099:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617221888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076103.764109:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076103.764113:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076103.764116:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617221888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076103.764124:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076103.764157:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076103.764195:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076103/real 1769076103] req@ffff9b0343dee680 x1855007617222016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076103.764219:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617222016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076103.764237:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076103.764244:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076103.764291:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076103.764304:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076103.764338:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076103.764354:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076103/real 1769076103] req@ffff9b0343defb80 x1855007617222144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076103.764371:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617222144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076103.764381:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076103.764385:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076103.764389:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076103.764397:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076103.764415:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076103.764426:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076103/real 1769076103] req@ffff9b0343dece00 x1855007617222272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076103.764440:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617222272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076103.764466:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076103.764470:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076103.764475:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076104.275420:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076104.659367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076104.659367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076104.659478:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076105.299403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076105.683386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076105.683404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076105.683438:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076106.323353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076106.707362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076106.707370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076106.707418:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076107.347371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076107.731361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076107.731390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076107.731447:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076108.371376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076108.755341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076108.755361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076108.755487:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076108.883365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076108.883380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076108.883388:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076108.883406:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3098 00000100:00080000:1.0:1769076108.883422:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076108.883431:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076108.883488:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617222400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076108.883540:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076108.883548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076108.883552:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076108.883560:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3098 00000100:00080000:1.0:1769076108.883566:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076108.883574:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076108.883589:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076108.883612:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617222400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076108.883615:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617222528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076108.883640:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076108.883645:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076108.883648:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076108.883654:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3098 00000100:00080000:1.0:1769076108.883659:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076108.883664:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076108.883670:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076108.883720:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617222656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076108.883735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076108.883740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076108.883745:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769076108.883749:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076108.883756:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3098 00000100:00080000:1.0:1769076108.883766:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076108.883773:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076108.883783:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076108.883800:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617222784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076108.883815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769076108.883821:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076108/real 1769076108] req@ffff9b0343deca80 x1855007617222400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076108.883823:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076108.883830:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076108.883844:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3098 00000100:00080000:1.0:1769076108.883854:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076108.883861:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076108.883868:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076108.883874:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617222400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076108.883889:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617222912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076108.883894:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076108.883912:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076108.883922:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617222400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076108.883955:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076108.883999:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076108.884015:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076108/real 1769076108] req@ffff9b0343deed80 x1855007617222528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076108.884032:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617222528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076108.884043:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076108.884047:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076108.884052:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076108.884059:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076108.884079:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076108.884090:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076108/real 1769076108] req@ffff9b0343deea00 x1855007617222656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076108.884105:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617222656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076108.884124:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076108.884127:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076108.884132:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076108.884139:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076108.884157:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076108.884167:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076108/real 1769076108] req@ffff9b0343def800 x1855007617222784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076108.884181:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617222784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076108.884191:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076108.884195:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076108.884198:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076108.884204:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076108.884221:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076108.884231:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076108/real 1769076108] req@ffff9b0343ded880 x1855007617222912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076108.884299:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617222912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076108.884309:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076108.884313:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076108.884318:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617222912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076109.395338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076109.779375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076109.779381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076109.779413:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076110.419370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076110.803384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076110.803416:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076110.803444:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076111.443378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076111.827380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076111.827381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076111.827462:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076112.467373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076112.851355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076112.851372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076112.851418:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076113.491375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076113.875363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076113.875376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076113.875618:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076114.003334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076114.003348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076114.003355:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076114.003369:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3103 00000100:00080000:1.0:1769076114.003383:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076114.003392:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076114.003436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617223040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076114.003473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076114.003479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076114.003482:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076114.003488:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3103 00000100:00080000:1.0:1769076114.003492:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076114.003496:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076114.003507:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076114.003521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617223168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076114.003533:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076114.003538:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076114.003540:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076114.003545:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3103 00000100:00080000:1.0:1769076114.003548:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076114.003552:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076114.003557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076114.003570:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617223296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076114.003581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076114.003585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076114.003614:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076114.003619:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3103 00000100:00080000:1.0:1769076114.003624:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076114.003629:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076114.003638:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076114.003643:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617223040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076114.003656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617223424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076114.003673:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076114.003677:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076114.003680:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076114.003685:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3103 00000100:00080000:1.0:1769076114.003689:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076114.003693:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076114.003700:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076114.003714:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617223552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076114.003817:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076114.003878:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076114.003944:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076114.003991:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076114.515349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076114.899352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076114.899357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076114.899417:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076115.539341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076115.923376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076115.923394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076115.923488:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076116.563347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076116.947409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076116.947424:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076116.947513:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769076117.075745:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769076117.075817:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769076117.075844:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076117.075923:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076117.075940:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076117.075955:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076117.075980:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:3.0:1769076117.076077:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076114/real 1769076117] req@ffff9b0343ded180 x1855007617223424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076117.076137:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617223424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076117.076158:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076117.076191:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076117.076200:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076117.076216:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076114/real 1769076117] req@ffff9b0343dedf80 x1855007617223552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076117.076282:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617223552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076117.076297:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076117.076305:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076117.076311:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076117.076324:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076114/real 1769076117] req@ffff9b0343def100 x1855007617223040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076117.076349:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617223040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076117.076362:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076117.076367:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076117.076373:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617223040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076117.076384:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076114/real 1769076117] req@ffff9b0343dee300 x1855007617223168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076117.076405:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617223168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076117.076420:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076117.076425:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076117.076434:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076117.076449:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076114/real 1769076117] req@ffff9b0343dec380 x1855007617223296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076117.076470:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617223296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076117.076484:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076117.076491:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076117.076497:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076117.587330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076117.971361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076117.971401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076117.971503:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076118.611360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076118.995351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076118.995388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076118.995489:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076119.123440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076119.123459:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076119.123467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076119.123484:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3108 00000100:00080000:3.0:1769076119.123501:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076119.123536:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076119.123597:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617223680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076119.123645:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076119.123651:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076119.123654:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076119.123659:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3108 00000100:00080000:3.0:1769076119.123665:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076119.123672:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076119.123687:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076119.123704:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617223808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076119.123721:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617223680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076119.123737:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076119.123765:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076119.123771:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076119.123785:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3108 00000100:00080000:3.0:1769076119.123812:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076119.123823:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769076119.123831:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769076119.123872:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076119.123913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617223936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076119.123935:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076119/real 1769076119] req@ffff9b0352a1a680 x1855007617223680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076119.123947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076119.123975:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076119.123986:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076119.124002:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3108 00000100:00080000:3.0:1769076119.124012:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076119.124017:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617223680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076119.124030:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076119.124049:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769076119.124059:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076119.124078:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076119.124088:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617224064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076119.124100:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617223680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076119.124146:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076119.124154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076119.124170:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076119.124182:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223808:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076119.124190:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3108 00000100:00080000:3.0:1769076119.124205:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076119.124216:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076119.124230:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769076119.124325:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076119.124370:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076119/real 1769076119] req@ffff9b0352a1b100 x1855007617223808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076119.124388:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617224192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076119.124421:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617223808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076119.124445:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076119.124454:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076119.124463:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076119.124479:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076119.124574:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076119.124601:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076119/real 1769076119] req@ffff9b0352a1ad80 x1855007617223936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076119.124631:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617223936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076119.124661:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076119.124670:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076119.124682:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617223936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076119.124698:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076119.124749:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076119.124771:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076119/real 1769076119] req@ffff9b0352a1b480 x1855007617224064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076119.124794:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617224064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076119.124809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076119.124815:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076119.124821:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076119.124858:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076119.124900:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076119.124923:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076119/real 1769076119] req@ffff9b0352a18a80 x1855007617224192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076119.124948:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617224192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076119.124965:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076119.124972:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076119.124980:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076119.635389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076120.019358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076120.019369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076120.019424:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076120.659375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076121.043380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076121.043447:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076121.043449:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076121.683368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076122.067384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076122.067399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076122.067564:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076122.707399:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076123.091412:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076123.091529:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076123.091688:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076123.731396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076124.115400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076124.115405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076124.115593:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076124.243528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076124.243542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076124.243547:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076124.243558:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3113 00000100:00080000:1.0:1769076124.243569:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076124.243579:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076124.243622:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617224320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076124.243659:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076124.243665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076124.243667:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076124.243672:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3113 00000100:00080000:1.0:1769076124.243677:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076124.243682:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076124.243694:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076124.243709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617224448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076124.243718:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617224320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076124.243733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076124.243739:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076124.243741:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076124.243746:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3113 00000100:00080000:1.0:1769076124.243751:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076124.243755:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076124.243761:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076124.243778:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617224576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076124.243790:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076124.243795:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076124.243798:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076124.243801:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3113 00000100:00080000:1.0:1769076124.243806:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076124.243814:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769076124.243817:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769076124.243822:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076124.243840:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617224704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076124.243852:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076124.243856:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076124.243858:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076124.243864:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3113 00000100:00080000:1.0:1769076124.243870:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076124.243877:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076124/real 1769076124] req@ffff9b0343dec000 x1855007617224320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076124.243879:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076124.243890:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076124.243913:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617224832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076124.243920:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617224320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076124.243950:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076124.243957:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076124.243963:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617224320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076124.243992:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076124.244017:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076124.244030:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076124/real 1769076124] req@ffff9b0343dec700 x1855007617224448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076124.244053:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617224448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076124.244064:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076124.244067:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076124.244071:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076124.244078:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076124.244094:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076124.244104:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076124/real 1769076124] req@ffff9b0343dec380 x1855007617224576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076124.244117:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617224576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076124.244126:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076124.244129:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076124.244133:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076124.244139:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076124.244155:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076124.244164:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076124/real 1769076124] req@ffff9b0343dee300 x1855007617224704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076124.244176:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617224704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076124.244192:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076124.244196:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076124.244200:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076124.244206:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076124.244221:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076124.244230:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076124/real 1769076124] req@ffff9b0343def100 x1855007617224832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076124.244292:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617224832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076124.244302:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076124.244306:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076124.244310:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617224832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076124.755402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076125.139359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076125.139376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076125.139437:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076125.779383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076126.163367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076126.163413:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076126.163509:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076126.803327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076127.187409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076127.187410:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076127.187546:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076127.827420:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076128.211378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076128.211431:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076128.211510:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076128.851393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076129.235409:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076129.235460:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076129.235496:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076129.363375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076129.363393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076129.363400:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076129.363416:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3118 00000100:00080000:1.0:1769076129.363431:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076129.363441:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076129.363493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617224960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076129.363549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076129.363558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076129.363561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076129.363571:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3118 00000100:00080000:1.0:1769076129.363579:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076129.363589:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076129.363605:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076129.363632:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617224960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076129.363636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617225088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076129.363670:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076129.363676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076129.363679:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076129.363686:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3118 00000100:00080000:1.0:1769076129.363692:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076129.363697:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076129.363705:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076129.363722:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617225216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076129.363735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076129.363746:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076129.363747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076129.363755:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076129.363763:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3118 00000100:00080000:1.0:1769076129.363770:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076129.363777:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076129.363785:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076129.363805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617225344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076129.363815:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076129/real 1769076129] req@ffff9b0343dedf80 x1855007617224960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076129.363826:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076129.363840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076129.363846:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076129.363856:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3118 00000100:00080000:1.0:1769076129.363862:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076129.363869:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617224960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076129.363875:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076129.363889:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769076129.363897:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076129.363910:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076129.363911:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617225472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076129.363924:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617224960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076129.363973:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076129.364003:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076129.364018:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076129/real 1769076129] req@ffff9b0343ded180 x1855007617225088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076129.364034:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617225088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076129.364045:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076129.364049:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076129.364053:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076129.364060:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076129.364078:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076129.364089:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076129/real 1769076129] req@ffff9b0343ded880 x1855007617225216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076129.364111:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617225216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076129.364121:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076129.364124:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076129.364128:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076129.364134:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076129.364152:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076129.364163:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076129/real 1769076129] req@ffff9b0343def800 x1855007617225344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076129.364177:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617225344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076129.364186:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076129.364189:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076129.364193:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076129.364199:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076129.364216:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076129.364226:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076129/real 1769076129] req@ffff9b0343deea00 x1855007617225472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076129.364278:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617225472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076129.364296:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076129.364300:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076129.364304:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076129.875417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076130.259311:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076130.259355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076130.259374:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076130.899397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076131.283384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076131.283426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076131.283569:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076131.923414:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076132.307408:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076132.307410:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076132.307502:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076132.947398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076133.331421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076133.331424:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076133.331643:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076133.971469:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076134.355342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076134.355386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076134.355574:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076134.483477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076134.483487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076134.483492:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076134.483503:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3123 00000100:00080000:1.0:1769076134.483514:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076134.483522:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076134.483561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617225600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076134.483590:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076134.483594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076134.483596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076134.483601:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3123 00000100:00080000:1.0:1769076134.483605:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076134.483610:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076134.483621:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076134.483638:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617225600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076134.483640:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617225728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076134.483659:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076134.483663:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076134.483665:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076134.483671:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3123 00000100:00080000:1.0:1769076134.483675:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076134.483680:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076134.483687:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076134.483699:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617225856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076134.483709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076134.483713:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076134.483715:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076134.483718:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3123 00000100:00080000:1.0:1769076134.483722:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076134.483729:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769076134.483733:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769076134.483736:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076134.483748:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617225984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076134.483759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076134.483763:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076134.483764:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076134.483767:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3123 00000100:00080000:1.0:1769076134.483772:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076134.483775:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076134.483779:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076134.483789:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617226112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076134.483931:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076134/real 1769076134] req@ffff9b0343deed80 x1855007617225600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076134.483980:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617225600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076134.484158:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076134.484167:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076134.484173:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617225600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076134.484343:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076134.484382:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076134.484400:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076134/real 1769076134] req@ffff9b0343deca80 x1855007617225728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076134.484420:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617225728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076134.484429:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076134.484432:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076134.484436:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076134.484442:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076134.484458:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076134.484467:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076134/real 1769076134] req@ffff9b0343dece00 x1855007617225856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076134.484478:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617225856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076134.484486:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076134.484488:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076134.484491:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076134.484515:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076134.484536:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076134.484547:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076134/real 1769076134] req@ffff9b0343defb80 x1855007617225984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076134.484561:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617225984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076134.484579:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076134.484583:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076134.484586:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617225984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076134.484593:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617226112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076134.484610:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076134.484620:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076134/real 1769076134] req@ffff9b0343dee680 x1855007617226112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076134.484633:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617226112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076134.484642:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076134.484646:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076134.484653:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617226112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076134.995330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076135.379315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076135.379338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076135.379357:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076136.019410:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076136.403345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076136.403404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076136.403530:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076137.043447:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076137.427322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076137.427323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076137.427394:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076138.067373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076138.451365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076138.451435:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076138.451485:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076139.091427:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076139.475393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076139.475449:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076139.475567:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076139.603321:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076139.603342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076139.603347:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076139.603356:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3128 00000100:00080000:1.0:1769076139.603364:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076139.603370:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076139.603406:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617226240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076139.603447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076139.603451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076139.603454:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076139.603460:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3128 00000100:00080000:1.0:1769076139.603466:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076139.603472:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076139.603483:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076139.603484:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617226240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076139.603500:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617226368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076139.603511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076139.603514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076139.603527:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076139.603531:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3128 00000100:00080000:1.0:1769076139.603535:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076139.603538:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076139.603542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076139.603553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617226496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076139.603562:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076139.603563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076139.603568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076139.603569:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076139.603573:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3128 00000100:00080000:1.0:1769076139.603577:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076139.603581:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076139.603585:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076139.603595:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617226624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076139.603602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769076139.603604:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076139/real 1769076139] req@ffff9b0343def480 x1855007617226240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076139.603606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076139.603609:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076139.603617:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3128 00000100:00080000:1.0:1769076139.603623:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076139.603626:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076139.603630:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076139.603632:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617226240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076139.603642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617226752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076139.603665:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076139.603672:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076139.603676:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617226240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076139.603698:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617226368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076139.603722:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076139.603737:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076139/real 1769076139] req@ffff9b0343ded500 x1855007617226368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076139.603753:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617226368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076139.603764:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076139.603769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076139.603773:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617226368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076139.603781:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617226496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076139.603799:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076139.603808:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076139/real 1769076139] req@ffff9b0343dee680 x1855007617226496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076139.603817:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617226496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076139.603834:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076139.603837:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076139.603839:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617226496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076139.603844:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617226624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076139.603856:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076139.603862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076139/real 1769076139] req@ffff9b0343defb80 x1855007617226624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076139.603871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617226624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076139.603877:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076139.603879:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076139.603881:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617226624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076139.603884:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617226752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076139.603894:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076139.603900:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076139/real 1769076139] req@ffff9b0343dece00 x1855007617226752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076139.603908:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617226752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076139.603914:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076139.603916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076139.603918:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617226752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076140.115384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076140.499333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076140.499378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076140.499530:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076141.139348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076141.523303:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076141.523349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076141.523470:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076142.163415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076142.547346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076142.547371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076142.547509:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076143.187376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076143.571361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076143.571393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076143.571560:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076144.211404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076144.595370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076144.595388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076144.595551:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076144.723507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076144.723519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076144.723526:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076144.723540:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3133 00000100:00080000:3.0:1769076144.723556:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076144.723566:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076144.723644:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617226880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076144.723709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076144.723713:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076144.723715:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076144.723720:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3133 00000100:00080000:3.0:1769076144.723724:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076144.723727:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076144.723739:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076144.723752:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617227008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076144.723761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076144.723765:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076144.723767:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076144.723771:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3133 00000100:00080000:3.0:1769076144.723773:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076144.723778:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076144.723782:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076144.723795:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617227136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076144.723806:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076144.723809:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076144.723813:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076144.723818:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3133 00000100:00080000:3.0:1769076144.723821:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076144.723824:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076144.723829:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076144.723843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617226880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076144.723851:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617227264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076144.723873:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076144.723876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076144.723877:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076144.723882:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3133 00000100:00080000:3.0:1769076144.723886:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076144.723891:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076144.723896:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076144.723907:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617227392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076144.723955:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076144.724025:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076144/real 1769076144] req@ffff9b0352a18700 x1855007617226880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076144.724060:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617226880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076144.724141:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076144.724152:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076144.724161:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617226880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076144.724188:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076144.724230:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076144.724287:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076144/real 1769076144] req@ffff9b0352a1aa00 x1855007617227008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076144.724305:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617227008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076144.724316:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076144.724321:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076144.724325:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076144.724334:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076144.724359:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076144.724380:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076144/real 1769076144] req@ffff9b0352a19f80 x1855007617227136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076144.724394:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617227136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076144.724413:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076144.724417:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076144.724421:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076144.724466:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076144.724487:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076144.724508:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076144/real 1769076144] req@ffff9b0352a1a300 x1855007617227264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076144.724522:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617227264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076144.724532:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076144.724535:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076144.724539:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076144.724546:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076144.724564:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076144.724574:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076144/real 1769076144] req@ffff9b0352a18e00 x1855007617227392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076144.724588:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617227392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076144.724609:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076144.724613:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076144.724617:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076145.235416:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076145.619363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076145.619363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076145.619398:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076146.259386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076146.643387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076146.643399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076146.643539:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076147.283368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076147.667339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076147.667381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076147.667457:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076148.307383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076148.691425:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076148.691445:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076148.691576:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076149.331438:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076149.715363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076149.715365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076149.715463:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076149.843418:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076149.843430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076149.843437:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076149.843449:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3139 00000100:00080000:1.0:1769076149.843461:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076149.843471:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076149.843516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617227520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076149.843556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076149.843561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076149.843563:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076149.843569:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3139 00000100:00080000:1.0:1769076149.843575:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076149.843581:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076149.843595:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076149.843617:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617227648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076149.843620:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617227520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076149.843648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076149.843653:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076149.843656:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076149.843665:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3139 00000100:00080000:1.0:1769076149.843672:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076149.843679:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076149.843690:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076149.843714:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617227776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076149.843735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076149.843742:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076149.843747:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769076149.843755:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076149.843764:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3139 00000100:00080000:1.0:1769076149.843779:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076149.843787:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076149.843797:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076149.843817:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617227904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076149.843836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076149/real 1769076149] req@ffff9b0343deca80 x1855007617227520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076149.843842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076149.843860:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076149.843869:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076149.843889:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3139 00000100:00080000:1.0:1769076149.843899:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076149.843902:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617227520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076149.843914:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769076149.843930:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769076149.843935:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769076149.843941:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076149.843953:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617227520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076149.843970:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617228032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076149.844011:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076149.844059:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076149.844079:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076149/real 1769076149] req@ffff9b0343deed80 x1855007617227648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076149.844095:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617227648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076149.844107:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076149.844112:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076149.844116:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076149.844125:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076149.844145:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076149.844156:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076149/real 1769076149] req@ffff9b0343deea00 x1855007617227776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076149.844170:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617227776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076149.844179:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076149.844183:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076149.844186:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076149.844193:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076149.844211:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076149.844221:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076149/real 1769076149] req@ffff9b0343def800 x1855007617227904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076149.844268:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617227904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076149.844279:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076149.844283:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076149.844287:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617227904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076149.844308:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076149.844331:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076149.844344:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076149/real 1769076149] req@ffff9b0343ded880 x1855007617228032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076149.844358:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617228032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076149.844367:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076149.844371:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076149.844374:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076150.355418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076150.739396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076150.739450:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076150.739455:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076151.379451:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076151.763368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076151.763403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076151.763551:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076152.403389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076152.787331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076152.787373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076152.787451:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076153.427345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076153.811374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076153.811396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076153.811473:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076154.451387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076154.835387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076154.835432:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076154.835471:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076154.963439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076154.963455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076154.963463:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076154.963478:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3144 00000100:00080000:3.0:1769076154.963494:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076154.963505:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076154.963564:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617228160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076154.963613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076154.963621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076154.963625:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076154.963633:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3144 00000100:00080000:3.0:1769076154.963641:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076154.963651:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076154.963668:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076154.963682:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617228160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076154.963697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617228288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076154.963724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076154.963731:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076154.963736:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076154.963745:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3144 00000100:00080000:3.0:1769076154.963769:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076154.963780:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769076154.963785:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769076154.963801:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076154.963835:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617228416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076154.963851:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076154/real 1769076154] req@ffff9b0352a18e00 x1855007617228160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076154.963868:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076154.963879:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076154.963884:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076154.963886:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617228160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076154.963931:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3144 00000100:00080000:3.0:1769076154.963952:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076154.963959:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076154.963967:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769076154.963974:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076154.963984:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617228160:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769076154.963988:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076154.964025:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617228544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076154.964033:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228288:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076154.964078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076154.964090:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076154.964092:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076154.964101:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076154.964127:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3144 00000100:00100000:2.0:1769076154.964128:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076154/real 1769076154] req@ffff9b0352a1a300 x1855007617228288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076154.964142:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076154.964154:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617228288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076154.964160:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769076154.964178:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769076154.964183:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769076154.964187:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076154.964199:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076154.964209:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076154.964282:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076154.964298:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076154/real 1769076154] req@ffff9b0352a19f80 x1855007617228416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076154.964309:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617228672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076154.964322:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617228416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076154.964364:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076154.964372:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076154.964377:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076154.964402:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076154.964425:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076154.964439:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076154/real 1769076154] req@ffff9b0352a1aa00 x1855007617228544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076154.964455:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617228544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076154.964465:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076154.964468:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076154.964472:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076154.964479:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076154.964495:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076154.964505:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076154/real 1769076154] req@ffff9b0352a18700 x1855007617228672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076154.964519:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617228672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076154.964529:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076154.964532:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076154.964535:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076155.475420:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076155.859382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076155.859419:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076155.859449:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076156.499506:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076156.883413:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076156.883470:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076156.883539:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076157.523347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076157.907314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076157.907344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076157.907417:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076158.547390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076158.931370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076158.931375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076158.931579:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076159.571356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076159.955303:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076159.955345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076159.955474:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076160.083455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076160.083463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076160.083468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076160.083476:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3149 00000100:00080000:1.0:1769076160.083486:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076160.083492:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076160.083526:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617228800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076160.083550:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076160.083554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076160.083555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076160.083559:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3149 00000100:00080000:1.0:1769076160.083561:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076160.083565:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076160.083573:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076160.083583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617228928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076160.083591:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617228800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076160.083596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076160.083600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076160.083601:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076160.083605:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3149 00000100:00080000:1.0:1769076160.083607:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076160.083611:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076160.083616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076160.083627:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617229056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076160.083635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076160.083637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076160.083639:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076160.083642:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3149 00000100:00080000:1.0:1769076160.083648:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076160.083650:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076160.083654:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769076160.083659:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076160.083667:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617229184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076160.083676:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076160.083679:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076160.083681:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076160.083683:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3149 00000100:00080000:1.0:1769076160.083686:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076160.083690:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076160.083694:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076160.083699:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076160/real 1769076160] req@ffff9b0343ded180 x1855007617228800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076160.083704:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617229312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076160.083736:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617228800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076160.083747:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076160.083752:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076160.083755:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617228800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076160.083775:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076160.083791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076160.083813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076160/real 1769076160] req@ffff9b0343dedf80 x1855007617228928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076160.083855:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617228928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076160.083866:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076160.083870:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076160.083874:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617228928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076160.083882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076160.083907:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076160.083919:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076160/real 1769076160] req@ffff9b0343def100 x1855007617229056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076160.083929:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617229056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076160.083941:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076160.083943:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076160.083946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076160.083950:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076160.083964:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076160.083971:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076160/real 1769076160] req@ffff9b0343dee300 x1855007617229184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076160.083980:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617229184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076160.083985:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076160.083987:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076160.083990:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076160.083994:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076160.084004:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076160.084011:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076160/real 1769076160] req@ffff9b0343dec380 x1855007617229312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076160.084019:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617229312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076160.084025:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076160.084027:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076160.084030:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076160.595334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076160.979385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076160.979389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076160.979437:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076161.619438:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076162.003421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076162.003759:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076162.004021:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076162.643384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076163.027386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076163.027547:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076163.027736:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076163.667336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076164.051529:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076164.052439:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076164.052489:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076164.691325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769076165.075771:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076165.076028:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076165.076169:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:0.0:1769076165.203439:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076165.203455:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076165.203462:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076165.203477:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3154 00000100:00080000:0.0:1769076165.203494:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076165.203505:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076165.203569:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617229440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076165.203609:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076165.203615:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076165.203617:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076165.203623:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3154 00000100:00080000:0.0:1769076165.203628:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076165.203633:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076165.203658:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076165.203675:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617229568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076165.203691:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076165.203695:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076165.203697:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076165.203704:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3154 00000100:00100000:2.0:1769076165.203713:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617229440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076165.203753:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076165.203766:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076165.203774:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076165.203798:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617229696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076165.203812:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076165.203816:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076165.203819:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076165.203824:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3154 00000100:00080000:0.0:1769076165.203830:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076165.203836:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076165.203842:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076165.203854:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617229824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076165.203869:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076165.203878:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076165.203879:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076165.203889:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076165.203899:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3154 00000100:00080000:0.0:1769076165.203906:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076165.203911:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076165.203918:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076165.203933:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617229952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076165.203973:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076165/real 1769076165] req@ffff9b0365a38a80 x1855007617229440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076165.204009:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617229440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076165.204027:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076165.204034:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076165.204041:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617229440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076165.204080:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076165.204115:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076165.204131:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076165/real 1769076165] req@ffff9b0365a39c00 x1855007617229568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076165.204146:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617229568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076165.204156:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076165.204160:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076165.204163:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076165.204172:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076165.204190:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076165.204199:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076165/real 1769076165] req@ffff9b0365a38e00 x1855007617229696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076165.204212:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617229696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076165.204220:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076165.204223:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076165.204226:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076165.204265:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076165.204286:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076165.204297:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076165/real 1769076165] req@ffff9b0365a3b480 x1855007617229824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076165.204310:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617229824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076165.204319:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076165.204323:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076165.204326:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076165.204332:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076165.204348:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076165.204357:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076165/real 1769076165] req@ffff9b0365a3aa00 x1855007617229952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076165.204370:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617229952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076165.204378:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076165.204381:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076165.204384:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617229952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076165.715379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769076166.099477:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076166.099491:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076166.099591:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076166.739417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076167.123472:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076167.123504:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076167.123570:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076167.763291:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076168.147369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076168.147429:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076168.147438:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076168.787369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076169.171343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076169.171509:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076169.171568:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076169.811345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076170.195340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076170.195373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076170.195412:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076170.323516:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076170.323550:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076170.323559:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076170.323578:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3159 00000100:00080000:3.0:1769076170.323598:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076170.323609:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076170.323714:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617230080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076170.323766:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076170.323773:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076170.323779:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076170.323790:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3159 00000100:00080000:3.0:1769076170.323801:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076170.323808:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076170.323826:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076170.323840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617230080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076170.323853:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617230208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076170.323879:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769076170.323941:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076170.323997:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076170/real 1769076170] req@ffff9b0352a18a80 x1855007617230080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076170.324017:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076170.324029:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076170.324048:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617230080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076170.324061:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076170.324066:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076170.324071:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617230080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076170.324100:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076170.324127:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076170.324141:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076170/real 1769076170] req@ffff9b0352a1b480 x1855007617230208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076170.324156:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617230208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076170.324165:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076170.324168:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076170.324171:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230208:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076170.324379:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3159 00000100:00080000:3.0:1769076170.324389:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076170.324399:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076170.324411:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076170.324438:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617230336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076170.324478:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076170.324486:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076170.324491:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076170.324511:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3159 00000100:00080000:3.0:1769076170.324521:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076170.324533:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076170.324548:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230336:192.168.204.160@tcp:38:kworker.0 00000008:00100000:3.0:1769076170.324548:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076170.324599:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617230464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076170.324619:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769076170.324665:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076170.324668:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076170.324673:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076170.324684:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3159 00000100:00080000:3.0:1769076170.324692:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076170.324697:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076170/real 1769076170] req@ffff9b0352a1ad80 x1855007617230336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076170.324704:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076170.324725:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076170.324736:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617230336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076170.324749:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617230592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076170.324752:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076170.324777:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076170.324796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076170.324823:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076170.324850:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076170.324861:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076170/real 1769076170] req@ffff9b0352a1b100 x1855007617230464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076170.324881:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617230464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076170.324888:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076170.324891:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076170.324894:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076170.324901:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076170.324916:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076170.324925:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076170/real 1769076170] req@ffff9b0352a1a680 x1855007617230592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076170.324936:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617230592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076170.324944:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076170.324947:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076170.324950:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076170.835339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076171.219301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076171.219303:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076171.219338:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076171.859366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076172.243359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076172.243466:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076172.243517:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076172.883341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076173.267301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076173.267364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076173.267374:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076173.907367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076174.291314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076174.291315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076174.291389:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076174.931340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076175.315338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076175.315342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076175.315404:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076175.443449:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076175.443465:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076175.443472:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076175.443488:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3164 00000100:00080000:3.0:1769076175.443506:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076175.443518:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076175.443609:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617230720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076175.443658:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076175.443664:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076175.443667:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076175.443673:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3164 00000100:00080000:3.0:1769076175.443677:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076175.443682:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076175.443696:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076175.443712:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617230848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076175.443727:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076175.443733:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076175.443736:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076175.443744:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3164 00000100:00080000:3.0:1769076175.443748:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076175.443754:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076175.443761:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076175.443774:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617230976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076175.443785:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617230720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076175.443793:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076175.443801:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076175.443803:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076175.443810:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3164 00000100:00080000:3.0:1769076175.443814:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076175.443819:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076175.443826:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076175.443841:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617231104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076175.443853:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076175.443857:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076175.443860:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076175.443864:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3164 00000100:00080000:3.0:1769076175.443869:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076175.443873:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076175.443879:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076175.443893:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617231232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076175.443928:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076175.443998:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076175/real 1769076175] req@ffff9b0352a18000 x1855007617230720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076175.444025:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617230720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076175.444041:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076175.444047:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076175.444053:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617230720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076175.444064:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076175.444091:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076175.444105:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076175/real 1769076175] req@ffff9b0352a19c00 x1855007617230848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076175.444120:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617230848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076175.444131:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076175.444135:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076175.444139:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076175.444177:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076175.444197:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076175.444208:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076175/real 1769076175] req@ffff9b0352a18380 x1855007617230976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076175.444224:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617230976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076175.444283:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076175.444288:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076175.444292:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617230976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076175.444300:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076175.444327:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076175.444341:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076175/real 1769076175] req@ffff9b0352a1b800 x1855007617231104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076175.444356:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617231104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076175.444367:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076175.444370:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076175.444375:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076175.444381:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076175.444399:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076175.444408:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076175/real 1769076175] req@ffff9b0352a19500 x1855007617231232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076175.444422:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617231232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076175.444441:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076175.444445:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076175.444448:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076175.955369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076176.339350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076176.339359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076176.339460:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076176.979350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076177.363321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076177.363377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076177.363456:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076178.003432:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076178.387357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076178.387397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076178.387437:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076179.027371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076179.411339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076179.411370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076179.411487:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076180.051411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076180.435344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076180.435413:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076180.435551:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076180.563378:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076180.563391:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076180.563397:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076180.563412:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3169 00000100:00080000:3.0:1769076180.563425:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076180.563437:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076180.563508:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617231360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076180.563558:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076180.563562:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076180.563564:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076180.563568:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3169 00000100:00080000:3.0:1769076180.563572:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076180.563576:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076180.563586:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076180.563598:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617231488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076180.563608:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076180.563612:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076180.563614:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076180.563619:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3169 00000100:00080000:3.0:1769076180.563622:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076180.563626:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076180.563631:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076180.563641:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617231616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076180.563650:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076180.563653:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076180.563655:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076180.563665:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3169 00000100:00100000:1.0:1769076180.563667:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617231360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076180.563669:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076180.563680:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076180.563687:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076180.563699:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617231744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076180.563709:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076180.563713:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076180.563715:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076180.563719:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3169 00000100:00080000:3.0:1769076180.563723:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076180.563726:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076180.563730:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076180.563740:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617231872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076180.563797:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076180.563839:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076180.563875:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076180.563902:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076181.075321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076181.459349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076181.459381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076181.459431:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076182.099345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076182.483339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076182.483422:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076182.483574:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076183.123359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076183.507385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076183.507394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076183.507509:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769076183.635628:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769076183.635681:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076183.635702:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076183.635754:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076183.635772:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076183.635804:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076183.635831:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076183.635904:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076180/real 1769076183] req@ffff9b0352a19c00 x1855007617231744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076183.635957:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617231744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076183.635977:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076183.635986:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076183.635994:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076183.636009:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076180/real 1769076183] req@ffff9b0352a18000 x1855007617231872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076183.636034:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617231872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076183.636050:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076183.636057:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076183.636064:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076183.636078:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076180/real 1769076183] req@ffff9b0352a19500 x1855007617231360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076183.636103:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617231360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076183.636117:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076183.636124:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076183.636130:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617231360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076183.636143:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076180/real 1769076183] req@ffff9b0352a1b800 x1855007617231488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076183.636169:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617231488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076183.636184:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076183.636190:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076183.636197:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076183.636209:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076180/real 1769076183] req@ffff9b0352a18380 x1855007617231616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076183.636230:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617231616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076183.636289:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076183.636297:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076183.636304:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617231616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076184.147301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076184.531351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076184.531359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076184.531441:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076185.171307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076185.555419:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076185.555433:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076185.555548:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076185.683350:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076185.683359:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076185.683364:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076185.683382:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3174 00000100:00080000:3.0:1769076185.683431:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076185.683440:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076185.683478:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617232000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076185.683503:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076185.683506:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076185.683507:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076185.683511:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3174 00000100:00080000:3.0:1769076185.683514:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076185.683517:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076185.683525:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076185.683534:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617232128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076185.683543:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076185.683549:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617232000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076185.683549:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076185.683554:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076185.683561:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3174 00000100:00080000:3.0:1769076185.683563:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076185.683566:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076185.683569:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076185.683578:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617232256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076185.683585:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076185.683587:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076185.683589:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076185.683592:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3174 00000100:00080000:3.0:1769076185.683595:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076185.683597:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076185.683600:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076185.683608:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617232384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076185.683615:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769076185.683616:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076185.683619:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076185.683621:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076185.683627:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3174 00000100:00080000:3.0:1769076185.683630:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076185.683633:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076185.683636:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076185.683644:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617232512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076185.683667:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076185/real 1769076185] req@ffff9b0352a1a680 x1855007617232000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076185.683689:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617232000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076185.683698:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076185.683701:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076185.683704:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617232000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076185.683723:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076185.683739:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076185.683751:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076185/real 1769076185] req@ffff9b0352a1b100 x1855007617232128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076185.683761:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617232128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076185.683767:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076185.683769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076185.683771:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076185.683775:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076185.683788:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076185.683794:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076185/real 1769076185] req@ffff9b0352a1ad80 x1855007617232256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076185.683802:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617232256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076185.683807:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076185.683809:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076185.683811:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076185.683818:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076185.683848:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076185.683861:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076185/real 1769076185] req@ffff9b0352a1b480 x1855007617232384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076185.683874:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617232384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076185.683883:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076185.683886:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076185.683889:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076185.683895:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076185.683915:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076185.683926:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076185/real 1769076185] req@ffff9b0352a18a80 x1855007617232512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076185.683938:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617232512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076185.683947:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076185.683950:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076185.683953:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076186.195340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076186.579332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076186.579453:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076186.579520:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076187.219400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076187.603411:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076187.603523:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076187.603570:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076188.244295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076188.627334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076188.627447:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076188.627606:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076189.268319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076189.651434:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076189.651532:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076189.651622:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076190.291371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076190.675364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076190.675430:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076190.675487:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076190.803428:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076190.803437:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076190.803441:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076190.803449:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3179 00000100:00080000:3.0:1769076190.803459:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076190.803465:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076190.803496:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617232640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076190.803520:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076190.803523:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076190.803524:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076190.803528:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3179 00000100:00080000:3.0:1769076190.803530:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076190.803533:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076190.803539:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076190.803571:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617232768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076190.803584:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076190.803589:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076190.803593:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076190.803599:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3179 00000100:00080000:3.0:1769076190.803604:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076190.803614:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076190.803616:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617232640:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769076190.803621:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076190.803642:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617232896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076190.803656:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076190.803660:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076190.803663:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076190.803668:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3179 00000100:00080000:3.0:1769076190.803673:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076190.803677:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076190.803684:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076190.803697:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617233024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076190.803709:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769076190.803719:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076190.803721:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076190.803728:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076190.803736:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3179 00000100:00080000:3.0:1769076190.803742:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076190.803748:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076190.803755:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076190.803773:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617233152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076190.803787:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076190/real 1769076190] req@ffff9b0352a18700 x1855007617232640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076190.803834:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617232640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076190.803849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076190.803855:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076190.803860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617232640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076190.803871:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076190.803898:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076190.803913:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076190/real 1769076190] req@ffff9b0352a1aa00 x1855007617232768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076190.803928:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617232768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076190.803938:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076190.803942:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076190.803946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076190.803983:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076190.803997:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076190.804004:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076190/real 1769076190] req@ffff9b0352a19f80 x1855007617232896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076190.804011:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617232896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076190.804016:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076190.804018:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076190.804020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617232896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076190.804023:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076190.804033:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076190.804038:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076190/real 1769076190] req@ffff9b0352a1a300 x1855007617233024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076190.804045:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617233024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076190.804062:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076190.804063:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076190.804065:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076190.804068:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076190.804077:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076190.804082:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076190/real 1769076190] req@ffff9b0352a18e00 x1855007617233152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076190.804094:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617233152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076190.804099:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076190.804101:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076190.804103:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076191.315462:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076191.699500:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076191.699505:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076191.699528:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076192.339353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076192.723396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076192.723426:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076192.723552:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076193.363477:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076193.747359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076193.747411:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076193.747468:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076194.387394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076194.771324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076194.771383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076194.771505:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076195.411392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076195.795358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076195.795411:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076195.795507:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076195.923435:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076195.923448:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076195.923455:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076195.923469:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3185 00000100:00080000:2.0:1769076195.923483:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076195.923494:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076195.923543:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617233280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076195.923586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076195.923592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076195.923595:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076195.923600:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3185 00000100:00080000:2.0:1769076195.923605:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076195.923609:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076195.923621:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076195.923635:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617233408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076195.923648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076195.923652:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076195.923654:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076195.923659:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3185 00000100:00080000:2.0:1769076195.923662:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076195.923666:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076195.923670:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076195.923681:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617233536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076195.923693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076195.923697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076195.923700:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076195.923705:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3185 00000100:00080000:2.0:1769076195.923709:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076195.923714:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076195.923722:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076195.923736:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617233280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076195.923737:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617233664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076195.923764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076195.923768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076195.923770:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076195.923775:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3185 00000100:00080000:2.0:1769076195.923780:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076195.923783:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076195.923789:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076195.923804:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617233792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076195.923840:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076195.923891:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076195/real 1769076195] req@ffff9b03482c5180 x1855007617233280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076195.923915:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617233280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076195.923929:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076195.923934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076195.923940:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617233280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076195.923949:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076195.923974:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076195.923988:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076195/real 1769076195] req@ffff9b03482c4700 x1855007617233408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076195.924003:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617233408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076195.924013:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076195.924017:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076195.924021:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076195.924027:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076195.924045:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076195.924056:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076195/real 1769076195] req@ffff9b03482c7100 x1855007617233536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076195.924069:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617233536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076195.924079:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076195.924082:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076195.924086:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076195.924115:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076195.924134:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076195.924144:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076195/real 1769076195] req@ffff9b03482c4000 x1855007617233664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076195.924167:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617233664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076195.924177:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076195.924180:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076195.924184:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076195.924191:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076195.924207:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076195.924292:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076195/real 1769076195] req@ffff9b03482c6a00 x1855007617233792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076195.924309:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617233792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076195.924320:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076195.924323:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076195.924327:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617233792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076196.435342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076196.819338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076196.819369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076196.819403:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076197.459448:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076197.843460:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076197.843500:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076197.843600:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076198.483366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076198.867370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076198.867388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076198.867449:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076199.507385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076199.891395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076199.891476:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076199.891524:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076200.531348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076200.915354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076200.915386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076200.915494:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076201.043425:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.043436:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076201.043442:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.043454:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3190 00000100:00080000:3.0:1769076201.043466:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.043476:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076201.043525:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617233920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.043564:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.043569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076201.043572:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.043577:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3190 00000100:00080000:3.0:1769076201.043581:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.043586:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076201.043599:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076201.043615:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617234048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.043630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.043635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076201.043637:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.043643:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3190 00000100:00080000:3.0:1769076201.043647:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.043652:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076201.043661:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617233920:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769076201.043662:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076201.043683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617234176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.043698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.043703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076201.043706:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.043711:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3190 00000100:00080000:3.0:1769076201.043716:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.043735:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076201.043743:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076201.043765:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076201.043771:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617234304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076201.043794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076201.043801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076201.043806:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076201.043815:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3190 00000100:00080000:3.0:1769076201.043825:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076201.043835:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076201.043843:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076201/real 1769076201] req@ffff9b0352a18e00 x1855007617233920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076201.043849:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076201.043883:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617234432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076201.043902:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617233920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076201.043920:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076201.043927:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076201.043934:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617233920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076201.043948:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076201.044001:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076201.044019:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076201/real 1769076201] req@ffff9b0352a1a300 x1855007617234048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076201.044035:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617234048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076201.044046:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076201.044050:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076201.044054:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076201.044085:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076201.044107:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076201.044119:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076201/real 1769076201] req@ffff9b0352a19f80 x1855007617234176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076201.044169:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617234176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076201.044188:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076201.044192:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076201.044196:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076201.044205:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076201.044272:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076201.044290:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076201/real 1769076201] req@ffff9b0352a1aa00 x1855007617234304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076201.044305:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617234304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076201.044315:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076201.044319:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076201.044323:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076201.044331:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076201.044354:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076201.044366:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076201/real 1769076201] req@ffff9b0352a18700 x1855007617234432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076201.044380:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617234432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076201.044389:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076201.044393:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076201.044398:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076201.555381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076201.939318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076201.939338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076201.939358:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076202.579387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076202.963351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076202.963398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076202.963514:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076203.603369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076203.987310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076203.987311:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076203.987404:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076204.627344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076205.011377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076205.011471:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076205.011489:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076205.651352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076206.035371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076206.035432:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076206.035510:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076206.163446:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076206.163464:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076206.163470:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076206.163488:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3195 00000100:00080000:1.0:1769076206.163505:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076206.163515:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076206.163561:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617234560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076206.163605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076206.163611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076206.163614:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076206.163620:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3195 00000100:00080000:1.0:1769076206.163627:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076206.163636:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076206.163654:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076206.163679:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617234688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076206.163696:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617234560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076206.163713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076206.163724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076206.163729:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076206.163739:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3195 00000100:00080000:1.0:1769076206.163748:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076206.163764:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076206.163776:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076206.163811:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617234816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076206.163828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076206.163832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076206.163836:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076206.163841:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3195 00000100:00080000:1.0:1769076206.163850:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076206.163859:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076206.163870:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076206.163896:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617234944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076206.163918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076206.163926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076206.163930:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076206.163940:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3195 00000100:00080000:1.0:1769076206.163949:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076206.163957:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076206.163969:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076206.163990:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617235072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076206.164340:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076206.164510:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076206/real 1769076206] req@ffff9b0343dec700 x1855007617234560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076206.164540:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617234560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076206.164556:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076206.164563:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076206.164569:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617234560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076206.164608:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076206.164643:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076206.164660:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076206/real 1769076206] req@ffff9b0343dec000 x1855007617234688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076206.164677:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617234688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076206.164688:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076206.164693:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076206.164698:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076206.164706:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076206.164726:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076206.164738:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076206/real 1769076206] req@ffff9b0343dec380 x1855007617234816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076206.164752:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617234816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076206.164762:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076206.164765:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076206.164769:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076206.164777:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076206.164797:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076206.164808:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076206/real 1769076206] req@ffff9b0343dee300 x1855007617234944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076206.164822:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617234944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076206.164833:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076206.164837:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076206.164841:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617234944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076206.164848:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076206.164866:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076206.164876:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076206/real 1769076206] req@ffff9b0343def100 x1855007617235072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076206.164889:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617235072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076206.164899:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076206.164903:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076206.164907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076206.675343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076207.059300:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076207.059306:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076207.059332:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076207.699343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076208.083344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076208.083374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076208.083481:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076208.723347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076209.107405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076209.107427:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076209.107460:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076209.747385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076210.131338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076210.131338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076210.131433:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076210.771320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076211.155372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076211.155412:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076211.155565:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076211.283430:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076211.283443:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076211.283451:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076211.283463:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3200 00000100:00080000:1.0:1769076211.283476:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076211.283485:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076211.283533:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617235200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076211.283584:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076211.283589:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076211.283592:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076211.283597:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3200 00000100:00080000:1.0:1769076211.283601:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076211.283606:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076211.283618:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076211.283636:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617235328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076211.283653:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076211.283666:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1769076211.283667:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617235200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076211.283673:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076211.283699:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3200 00000100:00080000:1.0:1769076211.283703:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076211.283708:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076211.283716:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076211.283732:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617235456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076211.283744:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076211.283748:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076211.283751:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076211.283757:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3200 00000100:00080000:1.0:1769076211.283766:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076211.283771:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076211.283777:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076211.283793:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617235584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076211.283808:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769076211.283812:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076211.283831:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076211.283838:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076211.283846:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3200 00000100:00080000:1.0:1769076211.283852:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076211.283857:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076211.283863:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076211.283881:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617235712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076211.283930:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076211/real 1769076211] req@ffff9b0343dedf80 x1855007617235200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076211.284009:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617235200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076211.284041:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076211.284049:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076211.284055:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617235200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076211.284094:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076211.284127:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076211.284142:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076211/real 1769076211] req@ffff9b0343ded180 x1855007617235328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076211.284156:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617235328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076211.284166:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076211.284170:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076211.284174:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076211.284180:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076211.284198:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076211.284208:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076211/real 1769076211] req@ffff9b0343ded880 x1855007617235456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076211.284219:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617235456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076211.284228:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076211.284231:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076211.284233:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076211.284240:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076211.284292:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076211.284303:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076211/real 1769076211] req@ffff9b0343def800 x1855007617235584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076211.284316:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617235584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076211.284325:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076211.284329:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076211.284333:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076211.284340:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076211.284357:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076211.284366:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076211/real 1769076211] req@ffff9b0343deea00 x1855007617235712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076211.284391:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617235712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076211.284424:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076211.284428:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076211.284431:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076211.795348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076212.179326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076212.179334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076212.179486:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076212.819409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076213.203294:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076213.203316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076213.203357:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076213.843334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076214.227358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076214.227360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076214.227433:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076214.867406:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076215.251437:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076215.251457:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076215.251570:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076215.891373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076216.275362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076216.275390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076216.275471:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076216.403354:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076216.403365:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076216.403371:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076216.403386:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3205 00000100:00080000:1.0:1769076216.403400:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076216.403410:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076216.403457:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617235840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076216.403495:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076216.403501:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076216.403503:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076216.403508:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3205 00000100:00080000:1.0:1769076216.403513:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076216.403517:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076216.403529:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076216.403542:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617235968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076216.403555:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076216.403560:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076216.403563:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076216.403568:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3205 00000100:00080000:1.0:1769076216.403573:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076216.403578:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076216.403584:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076216.403595:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617236096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076216.403608:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076216.403614:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1769076216.403618:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617235840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076216.403619:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076216.403631:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3205 00000100:00080000:1.0:1769076216.403639:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076216.403643:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076216.403649:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076216.403668:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617236224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076216.403681:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076216.403685:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076216.403688:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076216.403693:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3205 00000100:00080000:1.0:1769076216.403709:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076216.403714:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076216.403721:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076216.403738:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617236352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076216.403750:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076216.403811:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076216/real 1769076216] req@ffff9b0343deed80 x1855007617235840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076216.403836:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617235840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076216.403850:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076216.403856:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076216.403862:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617235840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076216.403911:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076216.403945:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076216.403962:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076216/real 1769076216] req@ffff9b0343deca80 x1855007617235968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076216.403979:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617235968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076216.403990:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076216.403994:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076216.403998:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617235968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076216.404027:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076216.404048:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076216.404059:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076216/real 1769076216] req@ffff9b0343dece00 x1855007617236096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076216.404082:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617236096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076216.404093:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076216.404097:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076216.404100:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076216.404107:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076216.404130:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076216.404149:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076216/real 1769076216] req@ffff9b0343defb80 x1855007617236224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076216.404174:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617236224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076216.404190:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076216.404197:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076216.404204:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076216.404215:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076216.404302:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076216.404319:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076216/real 1769076216] req@ffff9b0343dee680 x1855007617236352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076216.404335:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617236352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076216.404353:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076216.404357:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076216.404361:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076216.915365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076217.299328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076217.299343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076217.299365:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076217.939354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076218.323345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076218.323361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076218.323612:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076218.963312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076219.347314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076219.347317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076219.347382:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076219.987295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076220.371298:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076220.371315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076220.371387:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076221.011382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076221.395325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076221.395346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076221.395431:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076221.523364:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076221.523376:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076221.523382:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076221.523394:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3210 00000100:00080000:1.0:1769076221.523406:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076221.523415:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076221.523461:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617236480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076221.523500:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076221.523505:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076221.523507:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076221.523513:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3210 00000100:00080000:1.0:1769076221.523516:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076221.523521:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076221.523532:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076221.523548:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617236608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076221.523573:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769076221.523581:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617236480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076221.523582:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076221.523589:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076221.523605:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3210 00000100:00080000:1.0:1769076221.523609:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076221.523613:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076221.523623:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076221.523649:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617236736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076221.523666:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076221.523673:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076221.523678:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076221.523685:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3210 00000100:00080000:1.0:1769076221.523697:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769076221.523697:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076221.523714:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076221.523726:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076221.523752:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617236864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076221.523766:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076221.523771:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076221.523774:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076221.523806:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076221/real 1769076221] req@ffff9b0343ded500 x1855007617236480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076221.523821:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3210 00000100:00080000:1.0:1769076221.523834:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076221.523841:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076221.523851:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076221.523852:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617236480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076221.523874:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076221.523877:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617236992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076221.523886:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076221.523903:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617236480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076221.523946:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076221.523982:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076221.523999:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076221/real 1769076221] req@ffff9b0343def480 x1855007617236608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076221.524016:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617236608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076221.524035:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076221.524039:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076221.524044:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076221.524051:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076221.524070:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076221.524081:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076221/real 1769076221] req@ffff9b0343dee680 x1855007617236736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076221.524096:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617236736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076221.524106:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076221.524109:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076221.524112:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076221.524120:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076221.524139:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076221.524150:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076221/real 1769076221] req@ffff9b0343defb80 x1855007617236864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076221.524165:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617236864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076221.524181:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076221.524187:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076221.524194:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076221.524206:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076221.524329:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076221.524347:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076221/real 1769076221] req@ffff9b0343dece00 x1855007617236992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076221.524364:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617236992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076221.524374:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076221.524378:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076221.524382:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617236992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076222.035359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076222.419411:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076222.419411:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076222.419485:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076223.059351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076223.443375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076223.443414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076223.443514:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076224.083379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076224.467341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076224.467342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076224.467399:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076225.107365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076225.491345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076225.491349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076225.491416:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076226.131384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076226.515367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076226.515376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076226.515492:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076226.643368:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076226.643380:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076226.643385:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076226.643397:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3215 00000100:00080000:1.0:1769076226.643410:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076226.643419:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076226.643463:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617237120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076226.643501:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076226.643506:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076226.643508:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076226.643514:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3215 00000100:00080000:1.0:1769076226.643518:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076226.643522:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076226.643535:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076226.643548:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617237248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076226.643561:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076226.643566:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076226.643568:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076226.643572:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3215 00000100:00080000:1.0:1769076226.643575:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076226.643579:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076226.643584:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076226.643596:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617237376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076226.643607:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076226.643611:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076226.643613:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076226.643618:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3215 00000100:00080000:1.0:1769076226.643622:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076226.643626:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076226.643634:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076226.643637:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617237120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076226.643651:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617237504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076226.643664:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076226.643669:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076226.643672:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076226.643677:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3215 00000100:00080000:1.0:1769076226.643682:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076226.643687:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076226.643693:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076226.643730:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617237632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076226.643761:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076226.643813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076226/real 1769076226] req@ffff9b0343deca80 x1855007617237120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076226.643838:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617237120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076226.643851:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076226.643857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076226.643863:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617237120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076226.643872:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617237248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076226.643901:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076226.643915:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076226/real 1769076226] req@ffff9b0343deed80 x1855007617237248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076226.643930:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617237248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076226.643941:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076226.643945:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076226.643949:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617237248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076226.643956:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617237376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076226.643974:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076226.643984:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076226/real 1769076226] req@ffff9b0343deea00 x1855007617237376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076226.643998:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617237376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076226.644008:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076226.644011:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076226.644015:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617237376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076226.644045:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617237504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076226.644065:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076226.644076:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076226/real 1769076226] req@ffff9b0343def800 x1855007617237504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076226.644099:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617237504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076226.644109:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076226.644113:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076226.644117:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617237504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076226.644123:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617237632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076226.644140:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076226.644149:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076226/real 1769076226] req@ffff9b0343ded880 x1855007617237632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076226.644162:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617237632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076226.644172:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076226.644175:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076226.644178:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617237632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076227.155363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076227.539360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076227.539385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076227.539518:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076228.179371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076228.563397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076228.563403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076228.563575:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076229.203357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076229.587536:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076229.587685:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076229.587886:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076230.227352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076230.611536:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076230.611740:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076230.612049:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076231.251377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076231.635338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076231.635347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076231.635356:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076231.763310:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076231.763318:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076231.763322:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076231.763330:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3220 00000100:00080000:1.0:1769076231.763338:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076231.763343:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076231.763373:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617237760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076231.763412:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076231.763415:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076231.763416:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076231.763420:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3220 00000100:00080000:1.0:1769076231.763422:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076231.763425:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076231.763433:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076231.763442:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617237888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076231.763474:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076231.763479:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076231.763483:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076231.763491:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617237760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076231.763494:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3220 00000100:00080000:1.0:1769076231.763507:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076231.763514:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076231.763528:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076231.763547:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617238016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076231.763559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076231.763564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076231.763566:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076231.763571:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3220 00000100:00080000:1.0:1769076231.763575:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076231.763580:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076231.763593:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076231.763607:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617238144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076231.763642:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076231.763681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076231.763684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076231.763686:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076231.763690:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3220 00000100:00080000:1.0:1769076231.763694:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076231.763697:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076231.763701:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076231/real 1769076231] req@ffff9b0343ded180 x1855007617237760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076231.763704:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076231.763720:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617238272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076231.763730:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617237760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076231.763738:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076231.763742:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076231.763746:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617237760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076231.763765:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617237888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076231.763781:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076231.763788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076231/real 1769076231] req@ffff9b0343dedf80 x1855007617237888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076231.763796:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617237888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076231.763802:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076231.763804:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076231.763806:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617237888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076231.763809:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076231.763819:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076231.763824:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076231/real 1769076231] req@ffff9b0343def100 x1855007617238016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076231.763831:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617238016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076231.763840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076231.763843:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076231.763845:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076231.763848:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076231.763859:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076231.763866:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076231/real 1769076231] req@ffff9b0343dee300 x1855007617238144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076231.763874:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617238144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076231.763879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076231.763881:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076231.763882:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076231.763886:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076231.763895:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076231.763902:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076231/real 1769076231] req@ffff9b0343dec380 x1855007617238272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076231.763909:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617238272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076231.763914:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076231.763916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076231.763919:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076232.275399:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076232.659688:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076232.659739:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076232.659852:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076233.299427:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076233.683393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076233.683442:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076233.683479:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076234.323420:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076234.707351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076234.707424:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076234.707608:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076235.347404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076235.731345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076235.731374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076235.731471:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076236.371362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076236.755386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076236.755403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076236.755420:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076236.883419:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076236.883436:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076236.883444:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076236.883460:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3226 00000100:00080000:1.0:1769076236.883480:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076236.885432:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076236.885484:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617238400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076236.885524:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076236.885552:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076236.885557:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076236.885563:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3226 00000100:00080000:1.0:1769076236.885574:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076236.885580:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076236.885592:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076236.885612:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617238528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076236.885628:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076236.885633:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076236.885636:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076236.885642:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617238400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076236.885646:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3226 00000100:00080000:1.0:1769076236.885659:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076236.885669:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076236.885680:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076236.885707:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617238656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076236.885727:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076236.885734:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1769076236.885740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076236.885743:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076236.885757:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3226 00000100:00080000:1.0:1769076236.885768:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076236.885775:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076236.885794:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076236/real 1769076236] req@ffff9b0343dec000 x1855007617238400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076236.885800:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076236.885827:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617238400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076236.885836:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617238784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076236.885845:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076236.885861:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076236.885873:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617238400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076236.885876:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076236.885886:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076236.885889:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076236.885897:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3226 00000100:00080000:1.0:1769076236.885902:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076236.885908:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076236.885915:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238528:192.168.204.160@tcp:38:kworker.0 00000008:00100000:1.0:1769076236.885918:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076236.885941:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617238912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076236.885990:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076236.886009:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076236/real 1769076236] req@ffff9b0343dec700 x1855007617238528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076236.886024:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617238528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076236.886035:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076236.886039:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076236.886044:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076236.886052:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076236.886071:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076236.886081:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076236/real 1769076236] req@ffff9b0343dec380 x1855007617238656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076236.886094:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617238656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076236.886104:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076236.886107:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076236.886111:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076236.886117:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076236.886134:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076236.886143:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076236/real 1769076236] req@ffff9b0343dee300 x1855007617238784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076236.886155:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617238784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076236.886164:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076236.886167:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076236.886171:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076236.886188:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076236.886205:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076236.886216:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076236/real 1769076236] req@ffff9b0343def100 x1855007617238912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076236.886229:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617238912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076236.886273:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076236.886277:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076236.886281:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617238912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076237.395352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076237.779361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076237.779373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076237.779450:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076238.419340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076238.803371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076238.803388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076238.803532:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076239.443337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076239.827375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076239.827386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076239.827537:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076240.467531:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076240.851325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076240.851391:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076240.851418:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076241.491385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076241.875288:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076241.875340:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076241.875353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769076242.003512:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076242.003520:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076242.003524:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076242.003533:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3231 00000100:00080000:3.0:1769076242.003542:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076242.003549:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076242.003599:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617239040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076242.003636:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076242.003642:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076242.003645:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076242.003652:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3231 00000100:00080000:3.0:1769076242.003659:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076242.003667:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076242.003682:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076242.003705:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617239168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076242.003713:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617239040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076242.003740:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076242.003746:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076242.003749:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076242.003768:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3231 00000100:00080000:3.0:1769076242.003774:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076242.003780:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076242.003788:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076242.003816:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617239296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076242.003823:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076242.003827:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1769076242.003829:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076242.003830:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076242.003838:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3231 00000100:00080000:3.0:1769076242.003842:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076242.003846:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076242.003851:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076242.003888:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617239424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076242.003900:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076242/real 1769076242] req@ffff9b0352a18a80 x1855007617239040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076242.003928:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617239040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076242.003943:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076242.003951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076242.003960:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617239040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076242.004001:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239168:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076242.004030:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076242.004036:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076242.004040:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076242.004047:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3231 00000100:00080000:3.0:1769076242.004054:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076242.004062:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076242.004067:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076242.004079:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076242.004101:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076242/real 1769076242] req@ffff9b0352a1b480 x1855007617239168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076242.004103:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617239552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076242.004180:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617239168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076242.004188:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076242.004191:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076242.004195:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076242.004201:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076242.004358:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076242.004388:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076242/real 1769076242] req@ffff9b0352a1ad80 x1855007617239296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076242.004414:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617239296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076242.004432:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076242.004466:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076242.004475:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076242.004517:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076242.004541:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076242.004554:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076242/real 1769076242] req@ffff9b0352a1b100 x1855007617239424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076242.004584:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617239424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076242.004605:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076242.004611:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076242.004620:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076242.004634:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076242.004663:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076242.004678:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076242/real 1769076242] req@ffff9b0352a1a680 x1855007617239552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076242.004694:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617239552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076242.004707:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076242.004713:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076242.004720:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076242.515411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076242.899381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076242.899382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076242.899538:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076243.539405:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769076243.923428:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076243.923429:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076243.923440:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076244.563381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076244.947343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076244.947455:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076244.947565:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076245.587421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076245.971370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076245.971401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076245.971526:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076246.611396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076246.995378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076246.995383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076246.995430:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076247.123542:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076247.123557:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076247.123564:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076247.123576:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3236 00000100:00080000:0.0:1769076247.123590:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076247.123599:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076247.123644:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617239680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076247.123684:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076247.123690:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076247.123692:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076247.123698:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3236 00000100:00080000:0.0:1769076247.123702:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076247.123706:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076247.123717:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076247.123732:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617239808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076247.123745:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076247.123759:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076247.123761:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076247.123765:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3236 00000100:00080000:0.0:1769076247.123769:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076247.123773:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076247.123778:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076247.123790:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617239936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076247.123812:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076247.123817:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076247.123819:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076247.123825:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3236 00000100:00080000:0.0:1769076247.123833:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076247.123835:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617239680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076247.123841:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076247.123858:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076247.123874:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617240064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076247.123887:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076247.123892:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076247.123894:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076247.123899:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3236 00000100:00080000:0.0:1769076247.123921:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076247.123925:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076247.123931:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076247.123945:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617240192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076247.123985:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076247.124033:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076247.124067:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076247.124103:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076247.635336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076248.019452:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076248.019469:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076248.019506:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076248.659386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076249.043392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076249.043464:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076249.043506:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076249.683398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076250.067380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076250.067382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076250.067491:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769076250.195599:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:3.0:1769076250.195654:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769076250.195676:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769076250.195748:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769076250.195756:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769076250.195774:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769076250.195781:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076250.195831:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076247/real 1769076250] req@ffff9b0365a39500 x1855007617240064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076250.195870:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617240064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076250.195887:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076250.195893:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076250.195899:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076250.195909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076247/real 1769076250] req@ffff9b0365a39180 x1855007617240192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076250.195926:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617240192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076250.195937:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076250.195942:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076250.195946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076250.195956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076247/real 1769076250] req@ffff9b0365a3ad80 x1855007617239680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076250.195974:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617239680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076250.196027:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076250.196033:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076250.196038:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617239680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076250.196046:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076247/real 1769076250] req@ffff9b0365a3bb80 x1855007617239808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076250.196073:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617239808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076250.196085:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076250.196089:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076250.196093:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076250.196102:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076247/real 1769076250] req@ffff9b0365a3b100 x1855007617239936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076250.196119:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617239936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076250.196129:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076250.196133:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076250.196137:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617239936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076250.707340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076251.091398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076251.091425:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076251.091499:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076251.731359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076252.115415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076252.115422:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076252.115503:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076252.243394:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076252.243406:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076252.243410:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076252.243421:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3241 00000100:00080000:1.0:1769076252.243430:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076252.243437:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076252.243468:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617240320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076252.243500:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076252.243520:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076252.243523:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076252.243528:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3241 00000100:00080000:1.0:1769076252.243533:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076252.243538:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076252.243552:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076252.243576:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617240448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076252.243590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769076252.243594:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617240320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076252.243596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076252.243602:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076252.243629:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3241 00000100:00080000:1.0:1769076252.243634:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076252.243640:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076252.243647:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076252.243672:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617240576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076252.243692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769076252.243698:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076252.243701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076252.243706:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076252.243712:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3241 00000100:00080000:1.0:1769076252.243718:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076252.243724:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076252.243732:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076252.243750:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617240704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076252.243757:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076252/real 1769076252] req@ffff9b0343dedf80 x1855007617240320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076252.243766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076252.243777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076252.243787:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076252.243795:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3241 00000100:00080000:1.0:1769076252.243801:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076252.243806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617240320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076252.243811:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076252.243822:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1769076252.243830:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076252.243840:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617240832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076252.243841:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076252.243851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617240320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076252.243891:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076252.243924:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076252.243941:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076252/real 1769076252] req@ffff9b0343ded180 x1855007617240448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076252.243959:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617240448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076252.243978:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076252.243982:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076252.243987:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076252.243996:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076252.244024:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076252.244039:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076252/real 1769076252] req@ffff9b0343ded880 x1855007617240576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076252.244055:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617240576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076252.244065:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076252.244070:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076252.244074:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076252.244081:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076252.244107:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076252.244121:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076252/real 1769076252] req@ffff9b0343def800 x1855007617240704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076252.244229:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617240704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076252.244265:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076252.244287:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076252.244293:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076252.244301:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076252.244331:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076252.244345:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076252/real 1769076252] req@ffff9b0343deea00 x1855007617240832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076252.244359:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617240832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076252.244368:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076252.244371:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076252.244375:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617240832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076252.755383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076253.139381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076253.139392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076253.139546:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076253.779372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076254.163377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076254.163377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076254.163461:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076254.803340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076255.187340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076255.187474:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076255.187545:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076255.827480:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076256.211338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076256.211397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076256.211574:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076256.851371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076257.235372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076257.235383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076257.235433:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076257.363430:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076257.363447:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076257.363453:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076257.363468:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3246 00000100:00080000:3.0:1769076257.363483:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076257.363493:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076257.363561:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617240960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076257.363627:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076257.363633:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076257.363635:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076257.363641:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3246 00000100:00080000:3.0:1769076257.363646:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076257.363652:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076257.363665:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076257.363683:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617241088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076257.363700:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076257.363704:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617240960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076257.363706:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076257.363713:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076257.363730:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3246 00000100:00080000:3.0:1769076257.363734:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076257.363738:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076257.363745:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076257.363765:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617241216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076257.363778:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076257.363783:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076257.363786:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076257.363792:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3246 00000100:00080000:3.0:1769076257.363798:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076257.363802:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076257.363808:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076257.363823:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617241344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076257.363838:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076257.363840:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076257.363848:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076257.363851:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076257.363859:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3246 00000100:00080000:3.0:1769076257.363885:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076257.363891:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076257.363897:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076257.363915:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617241472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076257.363925:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076257/real 1769076257] req@ffff9b0352a18380 x1855007617240960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076257.363958:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617240960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076257.363978:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076257.363988:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076257.363996:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617240960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076257.364041:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076257.364100:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076257.364121:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076257/real 1769076257] req@ffff9b0352a1b800 x1855007617241088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076257.364138:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617241088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076257.364150:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076257.364154:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076257.364159:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076257.364167:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076257.364294:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076257.364321:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076257/real 1769076257] req@ffff9b0352a19500 x1855007617241216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076257.364355:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617241216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076257.364366:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076257.364369:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076257.364374:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076257.364381:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076257.364409:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076257.364423:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076257/real 1769076257] req@ffff9b0352a18000 x1855007617241344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076257.364445:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617241344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076257.364460:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076257.364465:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076257.364472:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076257.364483:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076257.364519:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076257.364538:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076257/real 1769076257] req@ffff9b0352a19c00 x1855007617241472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076257.364553:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617241472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076257.364570:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076257.364574:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076257.364599:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076257.875392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076258.259358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076258.259449:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076258.259575:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076258.899435:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076259.283384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076259.283410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076259.283492:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076259.923385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076260.307388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076260.307394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076260.307547:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076260.947327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076261.331334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076261.331391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076261.331527:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076261.971328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769076262.355375:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076262.355376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076262.355396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769076262.483328:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076262.483338:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076262.483342:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076262.483352:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3251 00000100:00080000:3.0:1769076262.483362:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076262.483369:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076262.483404:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617241600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076262.483432:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076262.483436:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076262.483439:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076262.483444:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3251 00000100:00080000:3.0:1769076262.483449:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076262.483454:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076262.483464:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076262.483479:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617241728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076262.483490:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076262.483495:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617241600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076262.483495:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076262.483497:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076262.483501:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3251 00000100:00080000:3.0:1769076262.483505:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076262.483509:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076262.483514:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076262.483525:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617241856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076262.483534:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076262.483538:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076262.483540:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076262.483543:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3251 00000100:00080000:3.0:1769076262.483546:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076262.483550:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076262.483555:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076262.483561:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076262.483564:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617241984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076262.483573:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076262.483576:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076262.483578:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076262.483582:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3251 00000100:00080000:3.0:1769076262.483586:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076262.483589:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076262.483594:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076262.483606:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076262/real 1769076262] req@ffff9b0352a19c00 x1855007617241600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076262.483606:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617242112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076262.483627:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617241600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076262.483639:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076262.483645:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076262.483650:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617241600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076262.483675:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076262.483698:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076262.483709:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076262/real 1769076262] req@ffff9b0352a18000 x1855007617241728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076262.483719:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617241728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076262.483732:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076262.483735:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076262.483737:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076262.483742:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076262.483754:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076262.483761:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076262/real 1769076262] req@ffff9b0352a19500 x1855007617241856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076262.483770:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617241856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076262.483776:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076262.483788:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076262.483791:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076262.483799:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076262.483813:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076262.483821:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076262/real 1769076262] req@ffff9b0352a1b800 x1855007617241984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076262.483830:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617241984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076262.483836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076262.483838:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076262.483841:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617241984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076262.483846:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617242112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076262.483857:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076262.483864:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076262/real 1769076262] req@ffff9b0352a18380 x1855007617242112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076262.483872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617242112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076262.483879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076262.483881:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076262.483883:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617242112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076262.995385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076263.379342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076263.379352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076263.379473:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076264.019311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076264.403344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076264.403363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076264.403510:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076265.043366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076265.427354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076265.427423:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076265.427494:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076266.067392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076266.451313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076266.451320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076266.451390:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076267.091341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076267.475375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076267.475378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076267.475421:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076267.603384:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076267.603397:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076267.603403:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076267.603416:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3256 00000100:00080000:3.0:1769076267.603429:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076267.603438:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076267.603487:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617242240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076267.603527:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076267.603532:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076267.603535:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076267.603540:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3256 00000100:00080000:3.0:1769076267.603544:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076267.603549:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076267.603560:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076267.603574:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617242368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076267.603589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076267.603594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076267.603598:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076267.603603:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3256 00000100:00080000:3.0:1769076267.603608:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076267.603613:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076267.603620:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076267.603628:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617242240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076267.603639:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617242496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076267.603654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076267.603659:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076267.603661:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076267.603667:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3256 00000100:00080000:3.0:1769076267.603671:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076267.603675:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076267.603682:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076267.603697:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617242624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076267.603709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076267.603713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076267.603715:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076267.603724:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3256 00000800:00000100:1.0:1769076267.603726:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076267.603731:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076267.603738:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076267.603745:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076267.603761:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617242752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076267.603794:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076267/real 1769076267] req@ffff9b0352a1a680 x1855007617242240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076267.603821:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617242240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076267.603836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076267.603842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076267.603847:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617242240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076267.603858:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617242368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076267.603884:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076267.603898:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076267/real 1769076267] req@ffff9b0352a1b100 x1855007617242368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076267.603913:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617242368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076267.603924:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076267.603927:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076267.603931:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617242368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076267.603961:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617242496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076267.603980:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076267.603992:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076267/real 1769076267] req@ffff9b0352a1ad80 x1855007617242496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076267.604006:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617242496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076267.604057:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076267.604061:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076267.604066:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617242496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076267.604075:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617242624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076267.604098:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076267.604112:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076267/real 1769076267] req@ffff9b0352a1b480 x1855007617242624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076267.604128:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617242624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076267.604138:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076267.604141:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076267.604145:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617242624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076267.604152:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617242752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076267.604169:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076267.604179:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076267/real 1769076267] req@ffff9b0352a18a80 x1855007617242752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076267.604193:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617242752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076267.604203:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076267.604206:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076267.604210:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617242752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076268.115393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076268.499370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076268.499390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076268.499474:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076269.139328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076269.523354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076269.523395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076269.523438:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076270.163373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076270.547397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076270.547417:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076270.547491:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076271.187361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076271.571361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076271.571365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076271.571505:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076272.211403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076272.595359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076272.595399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076272.595465:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076272.723417:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076272.723442:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076272.723464:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076272.723475:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3261 00000100:00080000:1.0:1769076272.723485:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076272.723495:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076272.723544:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617242880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076272.723608:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076272.723613:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076272.723615:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076272.723621:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3261 00000100:00080000:1.0:1769076272.723626:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076272.723631:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076272.723642:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076272.723660:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617243008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076272.723678:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769076272.723682:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617242880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076272.723685:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076272.723693:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076272.723704:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3261 00000100:00080000:1.0:1769076272.723711:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076272.723719:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076272.723745:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076272.723765:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617243136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076272.723782:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076272.723784:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076272.723792:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076272.723797:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076272.723805:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3261 00000100:00080000:1.0:1769076272.723813:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076272.723820:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076272.723845:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076272.723849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076272/real 1769076272] req@ffff9b0343deed80 x1855007617242880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076272.723870:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617243264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076272.723886:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617242880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076272.723893:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076272.723907:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076272.723918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076272.723952:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076272.723977:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769076272.723979:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3261 00000100:00100000:3.0:1769076272.723987:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617242880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076272.723991:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076272.724003:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076272.724013:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076272.724029:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076272.724033:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617243392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076272.724083:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076272.724106:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076272/real 1769076272] req@ffff9b0343deca80 x1855007617243008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076272.724129:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617243008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076272.724138:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076272.724141:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076272.724145:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076272.724151:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076272.724172:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076272.724181:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076272/real 1769076272] req@ffff9b0343dece00 x1855007617243136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076272.724192:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617243136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076272.724200:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076272.724203:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076272.724206:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076272.724211:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076272.724224:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076272.724232:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076272/real 1769076272] req@ffff9b0343defb80 x1855007617243264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076272.724242:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617243264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076272.724283:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076272.724286:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076272.724289:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076272.724306:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076272.724323:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076272.724333:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076272/real 1769076272] req@ffff9b0343dee680 x1855007617243392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076272.724344:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617243392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076272.724351:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076272.724355:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076272.724357:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076273.235396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076273.619381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076273.619387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076273.619496:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076274.259402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076274.643342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076274.643386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076274.643489:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076275.283330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076275.667343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076275.667363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076275.667371:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076276.307401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076276.691373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076276.691471:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076276.691525:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076277.331410:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076277.715386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076277.715393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076277.715438:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076277.843544:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076277.843564:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076277.843573:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076277.843597:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3267 00000100:00080000:3.0:1769076277.843620:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076277.843637:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076277.843713:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617243520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076277.843778:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076277.843789:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076277.843797:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076277.843811:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3267 00000100:00080000:3.0:1769076277.843821:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076277.843830:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076277.843878:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617243520:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769076277.843880:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076277.843925:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617243648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076277.843996:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076277.844058:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076277/real 1769076277] req@ffff9b0352a18700 x1855007617243520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076277.844094:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617243520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076277.844110:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076277.844117:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076277.844124:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617243520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076277.844413:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076277.844422:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076277.844426:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076277.844439:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3267 00000100:00080000:3.0:1769076277.844451:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076277.844463:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076277.844512:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076277.844533:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076277/real 1769076277] req@ffff9b0352a1aa00 x1855007617243648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076277.844550:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617243648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076277.844563:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076277.844568:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076277.844576:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243648:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076277.844588:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076277.844601:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076277.844625:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617243776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076277.844662:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076277.844668:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076277.844671:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076277.844676:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3267 00000100:00080000:3.0:1769076277.844684:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076277.844693:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076277.844704:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076277.844709:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076277.844736:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617243904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076277.844756:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076277.844771:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076277.844783:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076277/real 1769076277] req@ffff9b0352a19f80 x1855007617243776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076277.844785:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076277.844793:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076277.844811:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3267 00000100:00100000:1.0:1769076277.844820:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617243776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076277.844823:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076277.844862:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076277.844877:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769076277.844885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076277.844895:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076277.844907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076277.844909:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617244032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076277.844936:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076277.844971:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076277.844987:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076277/real 1769076277] req@ffff9b0352a1a300 x1855007617243904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076277.845001:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617243904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076277.845012:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076277.845016:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076277.845019:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617243904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076277.845032:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076277.845050:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076277.845061:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076277/real 1769076277] req@ffff9b0352a18e00 x1855007617244032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076277.845074:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617244032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076277.845084:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076277.845088:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076277.845093:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076278.355384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076278.739366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076278.739438:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076278.739522:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076279.379375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076279.763346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076279.763390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076279.763479:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076280.403339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076280.787364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076280.787404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076280.787434:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076281.427353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076281.811309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076281.811315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076281.811400:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076282.451348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076282.835340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076282.835341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076282.835438:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076282.963321:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076282.963331:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076282.963335:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076282.963346:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3272 00000100:00080000:3.0:1769076282.963356:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076282.963363:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076282.963403:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617244160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076282.963429:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076282.963432:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076282.963438:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076282.963441:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3272 00000100:00080000:3.0:1769076282.963445:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076282.963448:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076282.963455:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076282.963465:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617244288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076282.963473:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076282.963476:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076282.963477:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076282.963479:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3272 00000100:00080000:3.0:1769076282.963482:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076282.963484:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076282.963488:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076282.963494:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617244160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076282.963498:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617244416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076282.963509:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076282.963512:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076282.963513:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076282.963516:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3272 00000100:00080000:3.0:1769076282.963518:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076282.963520:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076282.963524:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076282.963533:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617244544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076282.963539:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076282.963541:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076282.963542:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076282.963544:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3272 00000100:00080000:3.0:1769076282.963547:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076282.963549:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076282.963552:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076282.963562:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617244672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076282.963564:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076282.963618:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076282/real 1769076282] req@ffff9b0352a18e00 x1855007617244160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076282.963653:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617244160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076282.963665:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076282.963671:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076282.963674:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617244160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076282.963681:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076282.963700:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076282.963710:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076282/real 1769076282] req@ffff9b0352a1a300 x1855007617244288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076282.963719:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617244288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076282.963726:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076282.963728:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076282.963731:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076282.963748:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076282.963787:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076282.963795:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076282/real 1769076282] req@ffff9b0352a19f80 x1855007617244416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076282.963804:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617244416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076282.963809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076282.963811:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076282.963813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076282.963817:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076282.963828:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076282.963833:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076282/real 1769076282] req@ffff9b0352a1aa00 x1855007617244544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076282.963841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617244544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076282.963846:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076282.963847:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076282.963849:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076282.963853:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076282.963863:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076282.963869:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076282/real 1769076282] req@ffff9b0352a18700 x1855007617244672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076282.963877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617244672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076282.963882:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076282.963884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076282.963886:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076283.475359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076283.859330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076283.859330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076283.859465:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076284.499391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076284.883418:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076284.883419:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076284.883562:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076285.523435:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076285.907390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076285.907450:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076285.907518:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076286.547384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076286.931406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076286.931493:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076286.931552:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076287.571435:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076287.955411:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076287.955414:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076287.955490:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076288.083440:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076288.083458:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076288.083465:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076288.083480:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3277 00000100:00080000:0.0:1769076288.083498:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076288.083508:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076288.083588:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617244800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076288.083633:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076288.083639:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076288.083642:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076288.083648:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3277 00000100:00080000:0.0:1769076288.083653:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076288.083658:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076288.083703:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076288.083728:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617244928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076288.083737:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617244800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076288.083767:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076288.083772:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076288.083775:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076288.083784:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3277 00000100:00080000:0.0:1769076288.083793:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076288.083801:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076288.083808:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076288.083826:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617245056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076288.083840:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769076288.083854:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076288.083857:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076288.083861:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076288.083868:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3277 00000100:00080000:0.0:1769076288.083875:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076288.083881:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076288.083889:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076288.083909:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617245184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076288.083926:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076288/real 1769076288] req@ffff9b0365a3b100 x1855007617244800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076288.083927:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076288.083942:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076288.083947:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076288.083957:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3277 00000100:00080000:0.0:1769076288.083971:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076288.083986:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076288.083994:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076288.084022:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617244800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076288.084023:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617245312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076288.084130:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076288.084141:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076288.084151:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617244800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076288.084191:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076288.084221:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076288.084271:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076288/real 1769076288] req@ffff9b0365a3bb80 x1855007617244928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076288.084292:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617244928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076288.084306:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076288.084312:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076288.084318:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617244928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076288.084329:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076288.084362:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076288.084378:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076288/real 1769076288] req@ffff9b0365a3ad80 x1855007617245056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076288.084391:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617245056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076288.084400:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076288.084404:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076288.084407:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076288.084414:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076288.084434:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076288.084443:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076288/real 1769076288] req@ffff9b0365a39180 x1855007617245184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076288.084455:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617245184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076288.084462:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076288.084465:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076288.084468:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076288.084473:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076288.084487:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076288.084495:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076288/real 1769076288] req@ffff9b0365a39500 x1855007617245312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076288.084506:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617245312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076288.084513:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076288.084516:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076288.084518:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076288.595435:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076288.979400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076288.979400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076288.979617:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076289.619417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076290.003342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076290.003409:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076290.003522:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076290.643336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076291.027403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076291.027406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076291.027483:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076291.667327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076292.051315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076292.051344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076292.051488:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076292.691375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076293.075392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076293.075398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076293.075509:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076293.203406:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076293.203421:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076293.203427:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076293.203439:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3282 00000100:00080000:3.0:1769076293.203452:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076293.203461:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076293.203521:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617245440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076293.203564:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076293.203569:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076293.203571:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076293.203577:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3282 00000100:00080000:3.0:1769076293.203606:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076293.203614:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076293.203639:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076293.203659:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617245568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076293.203675:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076293.203681:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076293.203685:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076293.203693:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617245440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076293.203694:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3282 00000100:00080000:3.0:1769076293.203706:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076293.203723:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076293.203729:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076293.203751:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617245696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076293.203764:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076293.203768:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076293.203771:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076293.203775:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3282 00000100:00080000:3.0:1769076293.203781:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076293.203785:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076293.203793:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076293.203803:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617245824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076293.203825:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076293.203830:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076293.203832:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076293.203837:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3282 00000100:00080000:3.0:1769076293.203842:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076293.203848:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076293.203853:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076293.203874:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617245952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076293.203875:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076293.203976:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076293/real 1769076293] req@ffff9b0352a18a80 x1855007617245440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076293.204003:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617245440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076293.204035:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076293.204041:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076293.204046:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617245440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076293.204084:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076293.204114:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076293.204128:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076293/real 1769076293] req@ffff9b0352a1b480 x1855007617245568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076293.204143:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617245568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076293.204152:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076293.204156:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076293.204160:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076293.204167:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076293.204184:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076293.204194:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076293/real 1769076293] req@ffff9b0352a1ad80 x1855007617245696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076293.204207:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617245696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076293.204295:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076293.204299:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076293.204303:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076293.204311:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076293.204338:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076293.204351:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076293/real 1769076293] req@ffff9b0352a1b100 x1855007617245824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076293.204363:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617245824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076293.204372:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076293.204375:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076293.204379:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076293.204385:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076293.204401:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076293.204409:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076293/real 1769076293] req@ffff9b0352a1a680 x1855007617245952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076293.204420:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617245952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076293.204429:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076293.204432:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076293.204436:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617245952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076293.715323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076294.099417:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076294.099439:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076294.099458:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076294.739320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076295.123446:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076295.123452:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076295.123514:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076295.763373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076296.147436:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076296.147466:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076296.147963:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076296.787354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076297.171414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076297.171751:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076297.171954:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076297.811361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076298.195436:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076298.195457:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076298.196056:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076298.323378:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076298.323392:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076298.323399:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076298.323413:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3287 00000100:00080000:3.0:1769076298.323426:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076298.323436:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076298.323486:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617246080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076298.323573:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076298.323578:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076298.323581:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076298.323587:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3287 00000100:00080000:3.0:1769076298.323592:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076298.323598:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076298.323610:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076298.323629:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617246208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076298.323643:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076298.323648:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076298.323651:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076298.323657:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3287 00000100:00080000:3.0:1769076298.323660:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076298.323665:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076298.323674:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076298.323686:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617246336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076298.323701:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076298.323709:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076298.323714:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076298.323717:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617246080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076298.323727:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3287 00000100:00080000:3.0:1769076298.323742:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076298.323750:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076298.323757:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076298.323786:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617246464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076298.323800:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076298.323804:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076298.323807:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076298.323814:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3287 00000100:00080000:3.0:1769076298.323819:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076298.323824:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076298.323830:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076298.323844:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617246592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076298.323879:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076298.323952:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076298/real 1769076298] req@ffff9b0352a18380 x1855007617246080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076298.323979:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617246080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076298.323995:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076298.324002:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076298.324007:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617246080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076298.324019:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076298.324046:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076298.324060:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076298/real 1769076298] req@ffff9b0352a1b800 x1855007617246208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076298.324075:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617246208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076298.324086:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076298.324091:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076298.324096:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076298.324137:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076298.324158:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076298.324169:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076298/real 1769076298] req@ffff9b0352a19500 x1855007617246336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076298.324183:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617246336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076298.324194:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076298.324198:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076298.324202:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076298.324209:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076298.324228:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076298.324238:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076298/real 1769076298] req@ffff9b0352a18000 x1855007617246464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076298.324335:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617246464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076298.324343:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076298.324346:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076298.324351:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076298.324361:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076298.324418:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076298.324443:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076298/real 1769076298] req@ffff9b0352a19c00 x1855007617246592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076298.324469:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617246592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076298.324486:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076298.324493:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076298.324523:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076298.835391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076299.219432:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076299.219678:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076299.219794:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076299.859350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076300.243384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076300.243486:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076300.243512:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076300.883364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076301.267356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076301.267510:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076301.267518:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076301.907400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076302.291394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076302.291460:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076302.291510:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076302.931354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076303.315471:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076303.315472:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076303.315572:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076303.443356:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076303.443369:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076303.443375:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076303.443387:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3292 00000100:00080000:3.0:1769076303.443400:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076303.443435:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076303.443496:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617246720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076303.443546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076303.443555:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076303.443559:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076303.443567:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3292 00000100:00080000:3.0:1769076303.443576:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076303.443584:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076303.443600:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076303.443629:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617246848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076303.443647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076303.443652:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076303.443656:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076303.443661:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3292 00000100:00080000:3.0:1769076303.443666:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076303.443672:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076303.443680:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076303.443695:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617246976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076303.443704:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617246720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076303.443720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076303.443726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076303.443728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076303.443734:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3292 00000100:00080000:3.0:1769076303.443737:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076303.443741:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076303.443750:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076303.443773:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617247104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076303.443784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076303.443789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076303.443791:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076303.443795:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3292 00000100:00080000:3.0:1769076303.443802:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076303.443807:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076303.443813:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076303.443828:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617247232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076303.443869:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076303.443943:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076303/real 1769076303] req@ffff9b0352a19c00 x1855007617246720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076303.443970:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617246720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076303.443986:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076303.444006:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076303.444016:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617246720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076303.444033:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076303.444087:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076303.444113:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076303/real 1769076303] req@ffff9b0352a18000 x1855007617246848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076303.444141:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617246848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076303.444158:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076303.444164:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076303.444168:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076303.444209:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076303.444309:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076303.444326:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076303/real 1769076303] req@ffff9b0352a19500 x1855007617246976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076303.444341:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617246976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076303.444352:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076303.444356:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076303.444360:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617246976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076303.444368:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076303.444391:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076303.444402:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076303/real 1769076303] req@ffff9b0352a1b800 x1855007617247104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076303.444459:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617247104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076303.444470:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076303.444474:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076303.444478:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076303.444487:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076303.444515:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076303.444528:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076303/real 1769076303] req@ffff9b0352a18380 x1855007617247232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076303.444543:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617247232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076303.444553:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076303.444557:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076303.444560:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076303.955332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076304.339362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076304.339459:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076304.339518:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076304.979353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076305.363358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076305.363495:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076305.363553:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076306.003348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769076306.387437:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076306.387441:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076306.387485:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076307.027432:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076307.411472:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076307.411483:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076307.411514:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076308.051397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076308.435392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076308.435492:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076308.435503:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769076308.563383:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076308.563396:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076308.563403:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076308.563418:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3297 00000100:00080000:3.0:1769076308.563432:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076308.563442:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076308.563492:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617247360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076308.563528:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076308.563533:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076308.563535:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076308.563541:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3297 00000100:00080000:3.0:1769076308.563545:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076308.563549:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076308.563561:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076308.563575:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617247488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076308.563588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076308.563593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076308.563595:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076308.563604:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3297 00000100:00080000:3.0:1769076308.563609:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076308.563613:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076308.563620:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076308.563631:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617247616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076308.563651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076308.563658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076308.563667:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076308.563672:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617247360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076308.563685:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3297 00000100:00080000:3.0:1769076308.563696:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076308.563706:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076308.563719:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076308.563750:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617247744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076308.563784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076308.563790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076308.563793:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076308.563798:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3297 00000100:00080000:3.0:1769076308.563808:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076308.563813:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076308.563827:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076308.563843:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076308.563876:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617247872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076308.563894:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076308/real 1769076308] req@ffff9b0352a1a680 x1855007617247360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076308.563950:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617247360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076308.563967:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076308.563974:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076308.563980:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617247360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076308.563992:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076308.564023:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076308.564039:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076308/real 1769076308] req@ffff9b0352a1b100 x1855007617247488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076308.564056:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617247488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076308.564067:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076308.564072:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076308.564076:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076308.564116:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076308.564137:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076308.564149:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076308/real 1769076308] req@ffff9b0352a1ad80 x1855007617247616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076308.564189:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617247616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076308.564217:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076308.564224:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076308.564231:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076308.564345:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076308.564385:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076308.564402:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076308/real 1769076308] req@ffff9b0352a1b480 x1855007617247744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076308.564418:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617247744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076308.564429:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076308.564433:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076308.564438:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076308.564446:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076308.564466:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076308.564478:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076308/real 1769076308] req@ffff9b0352a18a80 x1855007617247872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076308.564492:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617247872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076308.564502:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076308.564506:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076308.564511:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617247872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076309.075378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076309.459320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076309.459325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076309.459372:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076310.099393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076310.483363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076310.483372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076310.483444:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076311.123384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076311.507453:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076311.507462:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076311.507505:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076312.147393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076312.531401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076312.531422:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076312.531567:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076313.171349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076313.555376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076313.555476:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076313.555544:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076313.683419:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076313.683437:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076313.683445:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076313.683461:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3302 00000100:00080000:3.0:1769076313.683479:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076313.683491:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076313.683554:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617248000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076313.683599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076313.683605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076313.683607:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076313.683613:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3302 00000100:00080000:3.0:1769076313.683617:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076313.683621:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076313.683636:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076313.683654:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617248128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076313.683670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076313.683676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076313.683681:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076313.683686:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617248000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076313.683691:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3302 00000100:00080000:3.0:1769076313.683704:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076313.683709:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076313.683715:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076313.683732:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617248256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076313.683744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076313.683750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076313.683753:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076313.683758:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3302 00000100:00080000:3.0:1769076313.683762:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076313.683767:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076313.683773:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076313.683785:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617248384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076313.683795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076313.683801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076313.683804:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076313.683809:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3302 00000100:00080000:3.0:1769076313.683813:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076313.683817:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076313.683823:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076313.683838:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617248512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076313.683899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076313.683961:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076313.683996:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076313.684031:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076314.195349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076314.579410:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076314.579423:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076314.579550:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076315.219350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076315.603421:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076315.603519:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076315.603526:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076316.243329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076316.627369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076316.627424:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076316.627439:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769076316.755918:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769076316.755993:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076316.756022:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076316.756093:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076316.756108:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076316.756122:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076316.756136:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:3.0:1769076316.756305:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076313/real 1769076316] req@ffff9b0352a1aa00 x1855007617248128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076316.756350:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617248128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076316.756373:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076316.756384:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076316.756392:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076316.756406:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076313/real 1769076316] req@ffff9b0352a19f80 x1855007617248256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076316.756428:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617248256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076316.756445:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076316.756453:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076316.756463:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076316.756474:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076313/real 1769076316] req@ffff9b0352a1a300 x1855007617248384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076316.756519:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617248384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076316.756533:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076316.756541:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076316.756550:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076316.756564:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076313/real 1769076316] req@ffff9b0352a18e00 x1855007617248512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076316.756586:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617248512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076316.756600:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076316.756606:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076316.756612:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076316.756625:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076313/real 1769076316] req@ffff9b0352a18700 x1855007617248000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076316.756646:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617248000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076316.756661:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076316.756668:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076316.756675:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617248000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076317.267335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076317.651394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076317.651455:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076317.651557:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076318.291556:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076318.675351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076318.675379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076318.675483:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076318.803450:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076318.803464:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076318.803470:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076318.803484:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3307 00000100:00080000:0.0:1769076318.803496:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076318.803506:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076318.803550:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617248640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076318.803591:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076318.803596:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076318.803599:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076318.803604:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3307 00000100:00080000:0.0:1769076318.803608:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076318.803613:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076318.803624:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076318.803638:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617248768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076318.803651:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076318.803655:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076318.803658:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076318.803663:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3307 00000100:00080000:0.0:1769076318.803667:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076318.803671:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076318.803676:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076318.803686:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617248896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076318.803701:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076318.803706:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076318.803709:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076318.803716:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3307 00000100:00080000:0.0:1769076318.803720:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076318.803725:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076318.803733:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076318.803748:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617249024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076318.803762:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076318.803767:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076318.803769:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076318.803774:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3307 00000100:00080000:0.0:1769076318.803777:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076318.803789:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076318.803795:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076318.803796:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617248640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076318.803812:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617249152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076318.803965:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076318.804052:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076318/real 1769076318] req@ffff9b0365a3aa00 x1855007617248640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076318.804093:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617248640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076318.804115:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076318.804124:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076318.804132:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617248640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076318.804148:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076318.804472:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076318.804512:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076318/real 1769076318] req@ffff9b0365a3b480 x1855007617248768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076318.804542:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617248768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076318.804560:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076318.804807:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076318.804819:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076318.804836:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076318.804891:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076318.804919:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076318/real 1769076318] req@ffff9b0365a38e00 x1855007617248896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076318.804945:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617248896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076318.804963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076318.804970:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076318.804977:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617248896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076318.805037:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076318.805083:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076318.805107:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076318/real 1769076318] req@ffff9b0365a39c00 x1855007617249024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076318.805131:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617249024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076318.805147:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076318.805180:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076318.805188:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076318.805202:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076318.805290:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076318.805316:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076318/real 1769076318] req@ffff9b0365a38a80 x1855007617249152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076318.805340:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617249152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076318.805356:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076318.805362:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076318.805379:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076319.315344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076319.699355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076319.699392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076319.699620:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076320.339426:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076320.723360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076320.723377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076320.723469:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076321.363353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076321.747362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076321.747384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076321.747437:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076322.387299:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076322.771379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076322.771386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076322.771520:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076323.411378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076323.795406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076323.795406:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076323.795522:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076323.923427:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076323.923440:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076323.923445:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076323.923457:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3313 00000100:00080000:3.0:1769076323.923487:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076323.923495:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076323.923538:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617249280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076323.923578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076323.923583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076323.923585:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076323.923591:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3313 00000100:00080000:3.0:1769076323.923595:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076323.923600:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076323.923613:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076323.923629:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617249408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076323.923644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076323.923653:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617249280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076323.923653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076323.923657:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076323.923671:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3313 00000100:00080000:3.0:1769076323.923676:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076323.923680:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076323.923686:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076323.923709:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617249536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076323.923726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076323.923732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076323.923736:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076323.923743:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3313 00000800:00000100:1.0:1769076323.923751:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076323.923755:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076323.923768:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076323.923780:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076323.923807:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076323/real 1769076323] req@ffff9b0352a18700 x1855007617249280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076323.923810:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617249664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076323.923878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076323.923883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076323.923887:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076323.923890:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617249280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076323.923903:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3313 00000100:00080000:1.0:1769076323.923914:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076323.923917:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076323.923942:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769076323.923953:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076323.923954:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617249280:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769076323.923974:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076323.923994:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076323.924005:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617249792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076323.924031:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076323.924053:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076323/real 1769076323] req@ffff9b0352a18e00 x1855007617249408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076323.924104:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617249408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076323.924118:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076323.924122:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076323.924127:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076323.924136:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076323.924163:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076323.924177:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076323/real 1769076323] req@ffff9b0352a1a300 x1855007617249536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076323.924191:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617249536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076323.924201:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076323.924205:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076323.924209:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076323.924215:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076323.924285:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076323.924311:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076323/real 1769076323] req@ffff9b0352a19f80 x1855007617249664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076323.924335:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617249664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076323.924351:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076323.924358:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076323.924363:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076323.924387:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076323.924430:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076323.924443:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076323/real 1769076323] req@ffff9b0352a1aa00 x1855007617249792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076323.924464:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617249792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076323.924473:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076323.924476:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076323.924480:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617249792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076324.435340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076324.819361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076324.819399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076324.819514:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076325.459404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076325.843386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076325.843395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076325.843532:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076326.483373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076326.867414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076326.867414:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076326.867529:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076327.507340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076327.891433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076327.891454:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076327.891573:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076328.531375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076328.915384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076328.915388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076328.915532:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076329.043499:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076329.043513:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076329.043518:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076329.043531:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3318 00000100:00080000:2.0:1769076329.043543:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076329.043552:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076329.043601:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617249920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076329.043642:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076329.043647:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076329.043650:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076329.043655:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3318 00000100:00080000:2.0:1769076329.043659:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076329.043663:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076329.043674:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076329.043690:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617250048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076329.043704:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076329.043709:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076329.043712:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076329.043717:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3318 00000100:00080000:2.0:1769076329.043721:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076329.043728:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617249920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076329.043731:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076329.043746:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076329.043766:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617250176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076329.043778:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076329.043783:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076329.043785:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076329.043791:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3318 00000100:00080000:2.0:1769076329.043797:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076329.043802:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076329.043808:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076329.043820:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617250304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076329.043831:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076329.043837:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076329.043841:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769076329.043848:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076329.043853:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3318 00000100:00080000:2.0:1769076329.043861:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076329.043866:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076329.043874:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076329.043890:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617250432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076329.043930:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9b03482c5f80 x1855007617249920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076329.043957:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617249920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076329.043972:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076329.043977:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076329.044025:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617249920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076329.044037:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076329.044072:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076329.044090:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9b03482c7b80 x1855007617250048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076329.044108:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617250048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076329.044119:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076329.044123:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076329.044127:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076329.044157:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076329.044178:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076329.044190:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9b03482c4e00 x1855007617250176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076329.044205:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617250176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076329.044215:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076329.044219:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076329.044223:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076329.044231:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076329.044332:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076329.044356:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9b03482c5c00 x1855007617250304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076329.044375:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617250304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076329.044394:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076329.044398:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076329.044402:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076329.044410:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076329.044435:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076329.044447:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076329/real 1769076329] req@ffff9b03482c6680 x1855007617250432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076329.044461:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617250432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076329.044471:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076329.044474:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076329.044478:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076329.555346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076329.939372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076329.939374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076329.939468:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076330.579382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076330.963318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076330.963318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076330.963371:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076331.603308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076331.987355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076331.987386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076331.987420:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076332.627379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076333.011384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076333.011410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076333.011433:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076333.651414:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076334.035388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076334.035403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076334.035468:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076334.163451:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076334.163464:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076334.163470:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076334.163482:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3323 00000100:00080000:0.0:1769076334.163495:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076334.163504:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076334.163551:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617250560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076334.163589:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076334.163594:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076334.163597:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076334.163602:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3323 00000100:00080000:0.0:1769076334.163606:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076334.163611:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076334.163623:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076334.163641:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617250688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076334.163657:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076334.163665:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617250560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076334.163666:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076334.163673:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076334.163687:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3323 00000100:00080000:0.0:1769076334.163691:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076334.163697:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076334.163704:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076334.163719:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617250816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076334.163732:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076334.163737:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076334.163739:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076334.163744:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3323 00000100:00080000:0.0:1769076334.163749:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076334.163754:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076334.163760:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076334.163766:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076334.163780:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617250944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076334.163796:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076334.163800:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076334.163803:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076334.163808:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3323 00000100:00080000:0.0:1769076334.163814:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076334.163819:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076334.163828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076334/real 1769076334] req@ffff9b0365a38a80 x1855007617250560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769076334.163829:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076334.163852:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617251072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076334.163947:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617250560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076334.163963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076334.163969:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076334.163975:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617250560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076334.164003:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076334.164032:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076334.164047:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076334/real 1769076334] req@ffff9b0365a39c00 x1855007617250688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076334.164063:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617250688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076334.164074:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076334.164078:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076334.164082:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076334.164089:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076334.164107:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076334.164118:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076334/real 1769076334] req@ffff9b0365a38e00 x1855007617250816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076334.164132:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617250816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076334.164150:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076334.164154:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076334.164158:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076334.164165:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076334.164183:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076334.164193:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076334/real 1769076334] req@ffff9b0365a3b480 x1855007617250944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076334.164207:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617250944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076334.164217:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076334.164221:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076334.164224:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617250944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076334.164231:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076334.164292:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076334.164305:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076334/real 1769076334] req@ffff9b0365a3aa00 x1855007617251072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076334.164319:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617251072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076334.164329:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076334.164333:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076334.164338:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076334.675403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076335.059325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076335.059406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076335.059430:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076335.699315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769076336.083445:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076336.083446:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076336.083480:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076336.723333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076337.107384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076337.107395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076337.107544:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076337.747445:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076338.131360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076338.131364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076338.131541:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076338.771412:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076339.155377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076339.155378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076339.155539:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076339.283389:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076339.283407:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076339.283415:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076339.283433:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3328 00000100:00080000:3.0:1769076339.283449:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076339.283463:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076339.283523:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617251200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076339.283562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076339.283567:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076339.283570:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076339.283575:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3328 00000100:00080000:3.0:1769076339.283581:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076339.283586:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076339.283598:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076339.283615:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617251328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076339.283628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076339.283632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076339.283636:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076339.283642:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3328 00000100:00080000:3.0:1769076339.283646:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076339.283652:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076339.283658:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076339.283669:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617251456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076339.283684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076339.283690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769076339.283695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617251200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076339.283695:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076339.283710:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3328 00000100:00080000:3.0:1769076339.283715:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076339.283720:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076339.283727:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076339.283744:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617251584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076339.283756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076339.283761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076339.283764:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076339.283769:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3328 00000100:00080000:3.0:1769076339.283773:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076339.283779:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076339.283785:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076339.283795:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076339.283804:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617251712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076339.283888:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076339/real 1769076339] req@ffff9b0352a1aa00 x1855007617251200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076339.283915:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617251200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076339.283929:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076339.283935:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076339.283944:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617251200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076339.283959:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076339.284000:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076339.284023:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076339/real 1769076339] req@ffff9b0352a19f80 x1855007617251328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076339.284052:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617251328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076339.284070:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076339.284077:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076339.284085:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076339.284124:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076339.284153:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076339.284167:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076339/real 1769076339] req@ffff9b0352a1a300 x1855007617251456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076339.284183:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617251456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076339.284193:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076339.284198:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076339.284202:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076339.284209:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076339.284227:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076339.284297:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076339/real 1769076339] req@ffff9b0352a18e00 x1855007617251584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076339.284314:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617251584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076339.284325:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076339.284329:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076339.284333:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076339.284341:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076339.284365:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076339.284377:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076339/real 1769076339] req@ffff9b0352a18700 x1855007617251712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076339.284392:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617251712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076339.284402:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076339.284405:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076339.284409:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076339.795373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076340.179351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076340.179373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076340.179395:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076340.819419:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076341.203303:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076341.203347:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076341.203362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076341.843394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076342.227344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076342.227345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076342.227472:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076342.867422:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076343.251391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076343.251433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076343.251519:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076343.891432:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076344.275344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076344.275437:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076344.275454:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076344.403460:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076344.403474:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076344.403482:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076344.403498:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3333 00000100:00080000:0.0:1769076344.403516:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076344.403527:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076344.403604:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617251840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076344.403650:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076344.403656:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076344.403658:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076344.403664:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3333 00000100:00080000:0.0:1769076344.403669:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076344.403674:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076344.403687:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076344.403703:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617251968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076344.403716:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076344.403744:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076344.403747:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076344.403753:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3333 00000100:00080000:0.0:1769076344.403759:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076344.403765:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076344.403773:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076344.403791:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617252096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076344.403812:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617251840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076344.403817:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076344.403840:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076344.403843:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076344.403851:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3333 00000100:00080000:0.0:1769076344.403855:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076344.403861:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076344.403868:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076344.403885:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617252224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076344.403898:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076344.403903:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076344.403905:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076344.403911:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3333 00000100:00080000:0.0:1769076344.403918:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076344.403924:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076344.403930:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076344.403934:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076344.403955:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617252352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076344.404004:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076344/real 1769076344] req@ffff9b0365a39500 x1855007617251840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076344.404032:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617251840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076344.404047:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076344.404053:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076344.404059:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617251840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076344.404070:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076344.404098:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076344.404112:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076344/real 1769076344] req@ffff9b0365a39180 x1855007617251968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076344.404128:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617251968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076344.404140:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076344.404144:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076344.404149:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617251968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076344.404186:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076344.404224:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076344.404310:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076344/real 1769076344] req@ffff9b0365a3ad80 x1855007617252096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076344.404327:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617252096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076344.404339:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076344.404343:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076344.404348:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076344.404356:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076344.404385:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076344.404398:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076344/real 1769076344] req@ffff9b0365a3bb80 x1855007617252224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076344.404413:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617252224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076344.404422:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076344.404425:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076344.404433:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076344.404446:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076344.404481:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076344.404503:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076344/real 1769076344] req@ffff9b0365a3b100 x1855007617252352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076344.404528:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617252352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076344.404545:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076344.404551:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076344.404558:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076344.915401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076345.299382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076345.299393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076345.299438:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076345.939333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076346.323369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076346.323380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076346.323386:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076346.963389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076347.347379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076347.347391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076347.347502:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076347.987385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076348.371403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076348.371455:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076348.371543:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076349.011423:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076349.395374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076349.395392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076349.395510:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076349.523420:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076349.523432:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076349.523439:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076349.523453:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3338 00000100:00080000:3.0:1769076349.523466:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076349.523478:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076349.523525:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617252480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076349.523562:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076349.523568:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076349.523571:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076349.523578:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3338 00000100:00080000:3.0:1769076349.523585:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076349.523591:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076349.523606:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076349.523630:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617252480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076349.523632:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617252608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076349.523686:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076349.523692:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076349.523696:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076349.523703:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3338 00000100:00080000:3.0:1769076349.523711:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076349.523718:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076349.523727:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076349.523749:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617252736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076349.523760:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076349.523777:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076349.523783:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1769076349.523827:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076349/real 1769076349] req@ffff9b0352a18a80 x1855007617252480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076349.523852:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617252480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076349.523867:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076349.523874:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076349.523880:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617252480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076349.523888:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076349.523904:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3338 00000100:00080000:3.0:1769076349.523912:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076349.523918:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076349.523927:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076349.523932:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076349.523966:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076349.523981:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076349/real 1769076349] req@ffff9b0352a1b480 x1855007617252608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076349.523998:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617252608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076349.524010:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076349.524015:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076349.524019:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076349.524026:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076349.524047:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076349.524061:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076349/real 1769076349] req@ffff9b0352a1ad80 x1855007617252736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076349.524076:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617252736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076349.524087:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076349.524090:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076349.524094:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076349.524144:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617252864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076349.524167:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076349.524172:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076349.524176:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076349.524183:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3338 00000100:00100000:1.0:1769076349.524192:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076349.524298:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076349.524340:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076349/real 1769076349] req@ffff9b0352a1b100 x1855007617252864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076349.524364:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617252864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076349.524364:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076349.524381:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:1.0:1769076349.524388:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769076349.524395:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769076349.524398:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076349.524409:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076349.524433:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617252992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076349.524501:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076349.524532:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076349.524548:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076349/real 1769076349] req@ffff9b0352a1a680 x1855007617252992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076349.524563:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617252992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076349.524573:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076349.524577:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076349.524581:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617252992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076350.035407:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076350.419298:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076350.419327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076350.419347:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076351.059315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076351.443299:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076351.443338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076351.443347:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076352.083313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076352.467378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076352.467382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076352.467531:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076353.107404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076353.491347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076353.491356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076353.491452:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076354.131377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076354.515345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076354.515349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076354.515510:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076354.643360:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076354.643386:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076354.643392:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076354.643405:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3343 00000100:00080000:3.0:1769076354.643417:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076354.643426:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076354.643471:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617253120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076354.643508:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076354.643513:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076354.643516:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076354.643521:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3343 00000100:00080000:3.0:1769076354.643525:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076354.643530:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076354.643541:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076354.643595:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617253248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076354.643609:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076354.643613:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076354.643616:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076354.643621:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3343 00000100:00080000:3.0:1769076354.643626:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076354.643630:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076354.643637:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076354.643655:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617253120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076354.643655:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617253376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076354.643672:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076354.643677:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076354.643683:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076354.643689:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3343 00000100:00080000:3.0:1769076354.643692:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076354.643697:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076354.643704:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076354.643717:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617253504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076354.643728:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076354.643733:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076354.643735:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076354.643739:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3343 00000100:00080000:3.0:1769076354.643744:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076354.643749:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076354.643754:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076354.643763:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076354.643771:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617253632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076354.643843:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076354/real 1769076354] req@ffff9b0352a18380 x1855007617253120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076354.643877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617253120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076354.643887:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076354.643891:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076354.643895:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617253120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076354.643902:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617253248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076354.643923:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076354.643933:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076354/real 1769076354] req@ffff9b0352a1b800 x1855007617253248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076354.643943:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617253248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076354.643950:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076354.643952:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076354.643955:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617253248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076354.643974:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617253376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076354.643986:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076354.643993:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076354/real 1769076354] req@ffff9b0352a19500 x1855007617253376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076354.644001:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617253376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076354.644013:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076354.644016:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076354.644019:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617253376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076354.644023:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617253504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076354.644034:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076354.644040:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076354/real 1769076354] req@ffff9b0352a18000 x1855007617253504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076354.644061:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617253504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076354.644068:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076354.644070:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076354.644073:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617253504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076354.644077:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617253632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076354.644089:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076354.644096:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076354/real 1769076354] req@ffff9b0352a19c00 x1855007617253632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076354.644104:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617253632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076354.644110:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076354.644113:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076354.644116:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617253632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076355.155352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076355.539369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076355.539462:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076355.539486:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076356.179304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076356.563300:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076356.563300:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076356.563362:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076357.203386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076357.587355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076357.587420:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076357.587551:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076358.227354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076358.611391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076358.611475:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076358.611529:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076359.251365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076359.635362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076359.635386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076359.635438:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076359.763350:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076359.763362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076359.763369:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076359.763382:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3348 00000100:00080000:3.0:1769076359.763397:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076359.763407:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076359.763460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617253760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076359.763532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076359.763537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076359.763540:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076359.763545:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3348 00000100:00080000:3.0:1769076359.763550:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076359.763556:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076359.763568:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076359.763586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617253888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076359.763601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076359.763605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076359.763608:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076359.763613:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3348 00000100:00080000:3.0:1769076359.763617:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076359.763626:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076359.763631:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617253760:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769076359.763634:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076359.763654:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617254016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076359.763682:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076359.763687:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076359.763690:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076359.763700:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3348 00000100:00080000:3.0:1769076359.763706:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076359.763712:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076359.763726:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076359.763727:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076359.763752:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617254144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076359.763765:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076359.763771:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076359.763774:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076359.763781:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3348 00000100:00080000:3.0:1769076359.763786:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076359.763792:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076359/real 1769076359] req@ffff9b0352a19c00 x1855007617253760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076359.763797:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076359.763821:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076359.763832:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617253760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076359.763848:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617254272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076359.763849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076359.763866:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076359.763879:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617253760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076359.763894:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617253888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076359.763924:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076359.763939:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076359/real 1769076359] req@ffff9b0352a18000 x1855007617253888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076359.763955:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617253888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076359.763966:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076359.763970:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076359.763975:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617253888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076359.764003:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076359.764023:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076359.764035:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076359/real 1769076359] req@ffff9b0352a19500 x1855007617254016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076359.764049:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617254016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076359.764060:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076359.764064:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076359.764068:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076359.764075:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076359.764091:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076359.764102:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076359/real 1769076359] req@ffff9b0352a1b800 x1855007617254144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076359.764129:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617254144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076359.764139:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076359.764143:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076359.764147:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076359.764154:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076359.764173:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076359.764184:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076359/real 1769076359] req@ffff9b0352a18380 x1855007617254272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076359.764208:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617254272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076359.764217:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076359.764221:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076359.764224:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076360.275347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076360.659400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076360.659520:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076360.659679:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076361.299370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769076361.683512:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076361.683664:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076361.684145:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076362.323380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769076362.707550:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076362.707633:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076362.707742:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076363.347390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076363.731625:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076363.731653:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076363.731669:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076364.371468:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769076364.755555:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076364.755633:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076364.755734:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:1.0:1769076364.883466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076364.883479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076364.883485:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076364.883498:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3354 00000100:00080000:1.0:1769076364.883511:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076364.883520:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076364.883566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617254400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076364.883617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076364.883623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076364.883625:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076364.883631:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3354 00000100:00080000:1.0:1769076364.883636:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076364.883641:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076364.883655:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076364.883683:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617254528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076364.883699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769076364.883703:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617254400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076364.883721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076364.883725:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076364.883737:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3354 00000100:00080000:1.0:1769076364.883743:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076364.883749:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076364.883769:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076364.883797:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617254656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076364.883808:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076364.883833:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076364.883842:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076364.883846:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076364.883857:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3354 00000100:00080000:1.0:1769076364.883867:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076364.883870:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076364/real 1769076364] req@ffff9b0343def480 x1855007617254400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076364.883882:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076364.883906:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076364.883917:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617254400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076364.883935:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076364.883945:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617254784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076364.883949:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076364.883963:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617254400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076364.883981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076364.883987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076364.883991:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076364.883998:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3354 00000100:00080000:1.0:1769076364.884005:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076364.884008:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254528:192.168.204.160@tcp:38:kworker.0 00000100:00080000:1.0:1769076364.884017:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076364.884031:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076364.884053:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617254912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076364.884078:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076364.884115:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076364/real 1769076364] req@ffff9b0343ded500 x1855007617254528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076364.884144:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617254528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076364.884156:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076364.884160:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076364.884165:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076364.884173:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076364.884202:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076364.884215:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076364/real 1769076364] req@ffff9b0343dee680 x1855007617254656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076364.884230:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617254656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076364.884275:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076364.884280:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076364.884284:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076364.884292:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076364.884316:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076364.884330:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076364/real 1769076364] req@ffff9b0343defb80 x1855007617254784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076364.884345:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617254784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076364.884363:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076364.884367:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076364.884371:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076364.884432:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076364.884460:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076364.884474:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076364/real 1769076364] req@ffff9b0343dece00 x1855007617254912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076364.884489:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617254912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076364.884499:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076364.884503:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076364.884507:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617254912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076365.395399:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076365.779401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076365.779566:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076365.779684:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076366.419393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076366.803403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076366.803403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076366.803478:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076367.443406:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076367.827312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076367.827316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076367.827364:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076368.467309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076368.851361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076368.851361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076368.851435:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076369.491370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076369.875376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076369.875388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076369.875490:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076370.003401:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076370.003416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076370.003424:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076370.003437:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3359 00000100:00080000:3.0:1769076370.003449:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076370.003458:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076370.003503:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617255040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076370.003544:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076370.003550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076370.003553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076370.003559:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3359 00000100:00080000:3.0:1769076370.003564:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076370.003570:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076370.003583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076370.003602:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617255168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076370.003621:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617255040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076370.003637:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076370.003647:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076370.003652:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076370.003663:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3359 00000100:00080000:3.0:1769076370.003674:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076370.003683:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076370.003694:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769076370.003720:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076370.003725:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617255296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076370.003747:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076370.003752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076370.003755:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076370.003760:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3359 00000100:00080000:3.0:1769076370.003767:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076370.003772:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076370.003781:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076370.003800:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617255424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076370.003813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff9b0352a1a680 x1855007617255040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076370.003819:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076370.003837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076370.003843:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076370.003853:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3359 00000100:00080000:3.0:1769076370.003859:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076370.003864:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076370.003871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617255040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076370.003875:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076370.003895:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617255552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076370.003913:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076370.003925:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076370.003933:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617255040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076370.003967:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076370.004008:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076370.004025:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff9b0352a1b100 x1855007617255168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076370.004043:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617255168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076370.004054:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076370.004059:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076370.004063:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076370.004071:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076370.004105:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076370.004125:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff9b0352a1ad80 x1855007617255296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076370.004150:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617255296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076370.004169:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076370.004176:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076370.004184:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076370.004197:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076370.004230:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076370.004279:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff9b0352a1b480 x1855007617255424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076370.004336:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617255424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076370.004348:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076370.004352:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076370.004356:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076370.004365:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076370.004393:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076370.004407:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076370/real 1769076370] req@ffff9b0352a18a80 x1855007617255552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076370.004421:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617255552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076370.004432:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076370.004436:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076370.004440:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076370.515351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076370.899396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076370.899410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076370.899504:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076371.539481:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076371.923367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076371.923368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076371.923408:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076372.563374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076372.947364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076372.947402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076372.947486:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076373.587396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076373.971443:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076373.971446:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076373.971575:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076374.611392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076374.995363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076374.995372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076374.995430:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076375.123360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076375.123370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076375.123375:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076375.123385:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3364 00000100:00080000:3.0:1769076375.123396:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076375.123405:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076375.123445:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617255680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076375.123474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076375.123478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076375.123480:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076375.123484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3364 00000100:00080000:3.0:1769076375.123487:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076375.123490:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076375.123499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076375.123513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617255808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076375.123542:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076375.123548:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617255680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076375.123550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076375.123557:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076375.123567:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3364 00000100:00080000:3.0:1769076375.123574:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076375.123581:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076375.123589:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076375.123605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617255936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076375.123614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076375.123617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076375.123620:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076375.123624:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3364 00000100:00080000:3.0:1769076375.123629:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076375.123637:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076375.123638:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076375.123649:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076375.123661:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617256064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076375.123670:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076375.123674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076375.123676:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076375.123680:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3364 00000100:00080000:3.0:1769076375.123684:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076375.123690:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076375/real 1769076375] req@ffff9b0352a18700 x1855007617255680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076375.123691:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076375.123701:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076375.123718:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617256192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076375.123725:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617255680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076375.123745:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076375.123751:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076375.123757:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617255680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076375.123781:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076375.123802:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076375.123813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076375/real 1769076375] req@ffff9b0352a18e00 x1855007617255808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076375.123843:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617255808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076375.123852:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076375.123856:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076375.123859:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076375.123864:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076375.123879:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076375.123887:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076375/real 1769076375] req@ffff9b0352a1a300 x1855007617255936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076375.123899:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617255936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076375.123913:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076375.123916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076375.123919:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617255936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076375.123924:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076375.123937:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076375.123945:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076375/real 1769076375] req@ffff9b0352a19f80 x1855007617256064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076375.123954:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617256064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076375.123961:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076375.123964:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076375.123967:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076375.123972:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076375.123984:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076375.124011:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076375/real 1769076375] req@ffff9b0352a1aa00 x1855007617256192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076375.124038:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617256192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076375.124046:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076375.124050:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076375.124056:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076375.635403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076376.019358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076376.019393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076376.019546:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076376.659389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076377.043378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076377.043391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076377.043471:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076377.683379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076378.067391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076378.067394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076378.067583:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076378.707411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076379.091324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076379.091392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076379.091451:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076379.731440:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076380.115366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076380.115371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076380.115501:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076380.243378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076380.243405:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076380.243411:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076380.243425:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3369 00000100:00080000:0.0:1769076380.243438:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076380.243447:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076380.243488:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617256320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076380.243536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076380.243543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076380.243546:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076380.243553:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3369 00000100:00080000:0.0:1769076380.243560:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076380.243566:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076380.243581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076380.243603:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617256448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076380.243622:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076380.243627:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076380.243630:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076380.243635:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3369 00000100:00080000:0.0:1769076380.243640:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076380.243648:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076380.243649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617256320:192.168.204.160@tcp:250:kworker.0 00000008:00100000:0.0:1769076380.243655:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076380.243673:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617256576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076380.243696:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076380.243700:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076380.243703:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076380.243707:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3369 00000100:00080000:0.0:1769076380.243728:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076380.243732:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076380.243738:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076380.243749:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617256704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076380.243760:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076380.243764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076380.243767:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076380.243773:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3369 00000100:00080000:0.0:1769076380.243778:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076380.243788:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076380.243788:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256448:192.168.204.160@tcp:38:kworker.0 00000008:00100000:0.0:1769076380.243800:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076380.243820:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617256832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076380.243856:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076380.243880:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076380.243941:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076380.755321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076381.139376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076381.139377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076381.139538:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076381.779327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076382.163327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076382.163327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076382.163344:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076382.803316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076383.187304:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076383.187353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076383.187472:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769076383.315655:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769076383.315711:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769076383.315734:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076383.315783:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076383.315794:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076383.315803:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076383.315812:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076383.315879:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076380/real 1769076383] req@ffff9b0365a3ad80 x1855007617256576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076383.315929:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617256576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076383.315949:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076383.315959:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076383.315967:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076383.315981:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076380/real 1769076383] req@ffff9b0365a39180 x1855007617256704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076383.316007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617256704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076383.316023:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076383.316030:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076383.316036:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076383.316050:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076380/real 1769076383] req@ffff9b0365a39500 x1855007617256832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076383.316105:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617256832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076383.316120:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076383.316126:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076383.316133:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076383.316145:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076380/real 1769076383] req@ffff9b0365a3b100 x1855007617256320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076383.316166:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617256320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076383.316180:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076383.316186:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076383.316191:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617256320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076383.316202:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076380/real 1769076383] req@ffff9b0365a3bb80 x1855007617256448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076383.316224:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617256448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076383.316278:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076383.316285:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076383.316291:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617256448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076383.827333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076384.211371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076384.211399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076384.211466:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076384.851526:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076385.235353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076385.235362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076385.235531:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076385.363377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076385.363398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076385.363404:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076385.363418:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3374 00000100:00080000:0.0:1769076385.363434:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076385.363444:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076385.363490:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617256960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076385.363528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076385.363533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076385.363537:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076385.363542:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3374 00000100:00080000:0.0:1769076385.363549:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076385.363556:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076385.363572:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076385.363596:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617257088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076385.363617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076385.363625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076385.363628:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076385.363636:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3374 00000100:00080000:0.0:1769076385.363644:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076385.363653:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076385.363674:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617256960:192.168.204.160@tcp:250:kworker.0 00000008:00100000:0.0:1769076385.363679:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076385.363712:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617257216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076385.363735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076385.363742:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076385.363746:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076385.363754:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3374 00000100:00080000:0.0:1769076385.363777:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076385.363791:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769076385.363793:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769076385.363808:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076385.363837:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617257344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076385.363868:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076385.363879:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769076385.363881:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076385/real 1769076385] req@ffff9b0365a3aa00 x1855007617256960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076385.363884:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076385.363903:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3374 00000100:00080000:0.0:1769076385.363917:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076385.363934:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076385.363945:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617256960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769076385.363962:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769076385.363988:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769076385.364001:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617257472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076385.364003:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076385.364022:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617256960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076385.364098:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076385.364154:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076385.364184:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076385/real 1769076385] req@ffff9b0365a3b480 x1855007617257088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076385.364213:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617257088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076385.364313:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076385.364325:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076385.364333:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076385.364384:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076385.364438:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076385.364466:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076385/real 1769076385] req@ffff9b0365a38e00 x1855007617257216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076385.364493:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617257216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076385.364513:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076385.364521:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076385.364528:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076385.364542:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076385.364605:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076385.364631:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076385/real 1769076385] req@ffff9b0365a39c00 x1855007617257344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076385.364658:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617257344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076385.364674:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076385.364682:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076385.364691:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076385.364705:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076385.364750:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076385.364772:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076385/real 1769076385] req@ffff9b0365a38a80 x1855007617257472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076385.364795:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617257472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076385.364810:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076385.364816:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076385.364822:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076385.875359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076386.259386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076386.259390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076386.259524:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076386.899485:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076387.283396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076387.283417:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076387.283457:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076387.923443:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076388.307378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076388.307386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076388.307533:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076388.947398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076389.331358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076389.331385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076389.331534:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076389.971409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076390.355395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076390.355395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076390.355506:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076390.483427:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076390.483440:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076390.483447:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076390.483459:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3379 00000100:00080000:1.0:1769076390.483473:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076390.483482:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076390.483528:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617257600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076390.483571:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076390.483578:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076390.483581:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076390.483586:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3379 00000100:00080000:1.0:1769076390.483592:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076390.483598:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076390.483611:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076390.483627:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617257728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076390.483642:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617257600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076390.483647:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076390.483659:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076390.483662:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076390.483669:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3379 00000100:00080000:1.0:1769076390.483675:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076390.483680:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076390.483687:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076390.483703:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617257856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076390.483715:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076390.483721:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076390.483724:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076390.483729:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3379 00000100:00080000:1.0:1769076390.483734:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076390.483738:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769076390.483748:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769076390.483751:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076390.483784:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617257984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076390.483798:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076390.483802:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1769076390.483811:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076390/real 1769076390] req@ffff9b0343deca80 x1855007617257600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076390.483812:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076390.483827:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3379 00000100:00080000:1.0:1769076390.483837:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076390.483848:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076390.483856:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076390.483863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617257600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076390.483879:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617258112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076390.483885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076390.483904:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076390.483914:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617257600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076390.483973:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076390.484008:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076390.484025:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076390/real 1769076390] req@ffff9b0343deed80 x1855007617257728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076390.484042:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617257728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076390.484053:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076390.484058:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076390.484063:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076390.484071:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076390.484090:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076390.484101:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076390/real 1769076390] req@ffff9b0343deea00 x1855007617257856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076390.484116:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617257856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076390.484126:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076390.484129:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076390.484134:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076390.484140:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076390.484157:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076390.484167:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076390/real 1769076390] req@ffff9b0343def800 x1855007617257984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076390.484180:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617257984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076390.484190:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076390.484194:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076390.484198:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617257984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076390.484205:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617258112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076390.484221:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076390.484266:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076390/real 1769076390] req@ffff9b0343ded880 x1855007617258112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076390.484295:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617258112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076390.484311:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076390.484317:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076390.484324:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617258112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076390.995345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076391.379445:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076391.379447:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076391.379497:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076392.019444:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076392.403460:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076392.403468:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076392.403555:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076393.043385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076393.427373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076393.427425:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076393.427608:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076394.067418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076394.451367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076394.451425:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076394.451546:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076395.091400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076395.475297:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076395.475305:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076395.475377:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076395.603441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076395.603455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076395.603461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076395.603473:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3384 00000100:00080000:0.0:1769076395.603486:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076395.603495:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076395.603542:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617258240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076395.603582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076395.603587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076395.603590:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076395.603597:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3384 00000100:00080000:0.0:1769076395.603601:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076395.603607:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076395.603619:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076395.603635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617258368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076395.603650:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076395.603656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076395.603658:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076395.603663:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3384 00000100:00080000:0.0:1769076395.603667:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076395.603671:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076395.603678:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076395.603683:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617258240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076395.603696:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617258496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076395.603714:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076395.603718:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076395.603721:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076395.603726:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3384 00000100:00080000:0.0:1769076395.603731:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076395.603735:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076395.603741:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076395.603758:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617258624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076395.603771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076395.603776:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076395.603779:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076395.603784:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3384 00000100:00080000:0.0:1769076395.603789:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076395.603793:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076395.603799:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769076395.603808:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076395.603823:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617258752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076395.603907:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076395/real 1769076395] req@ffff9b0365a38a80 x1855007617258240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076395.603948:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617258240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076395.603970:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076395.603979:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076395.603987:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617258240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076395.603997:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617258368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076395.604034:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076395.604052:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076395/real 1769076395] req@ffff9b0365a39c00 x1855007617258368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076395.604078:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617258368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076395.604089:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076395.604093:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076395.604097:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617258368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076395.604127:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617258496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076395.604148:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076395.604160:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076395/real 1769076395] req@ffff9b0365a38e00 x1855007617258496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076395.604175:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617258496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076395.604184:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076395.604188:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076395.604192:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617258496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076395.604199:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617258624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076395.604216:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076395.604227:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076395/real 1769076395] req@ffff9b0365a3b480 x1855007617258624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076395.604322:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617258624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076395.604342:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076395.604346:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076395.604350:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617258624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076395.604357:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617258752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076395.604390:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076395.604404:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076395/real 1769076395] req@ffff9b0365a3aa00 x1855007617258752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076395.604418:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617258752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076395.604428:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076395.604432:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076395.604436:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617258752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076396.115457:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076396.499321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076396.499389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076396.499432:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076397.139351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076397.523405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076397.523405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076397.523514:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076398.163390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076398.547369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076398.547427:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076398.547524:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076399.187404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076399.571358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076399.571383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076399.571530:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076400.211375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076400.595410:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076400.595490:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076400.595503:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076400.723426:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076400.723437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076400.723443:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076400.723455:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3389 00000100:00080000:3.0:1769076400.723468:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076400.723477:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076400.723523:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617258880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076400.723578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076400.723584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076400.723586:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076400.723592:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3389 00000100:00080000:3.0:1769076400.723597:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076400.723602:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076400.723614:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076400.723632:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617259008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076400.723648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076400.723657:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617258880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076400.723658:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076400.723665:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076400.723679:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3389 00000100:00080000:3.0:1769076400.723683:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076400.723688:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076400.723694:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076400.723710:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617259136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076400.723724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076400.723729:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076400.723731:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076400.723737:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3389 00000100:00080000:3.0:1769076400.723743:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076400.723747:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076400.723755:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076400.723770:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076400.723773:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617259264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076400.723823:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076400.723829:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076400.723832:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076400.723839:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3389 00000100:00080000:3.0:1769076400.723846:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076400.723852:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076400/real 1769076400] req@ffff9b0352a1aa00 x1855007617258880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076400.723857:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076400.723872:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076400.723897:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617258880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076400.723902:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617259392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076400.723941:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076400.723953:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076400.723961:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617258880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076400.723994:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076400.724024:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076400.724039:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076400/real 1769076400] req@ffff9b0352a19f80 x1855007617259008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076400.724056:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617259008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076400.724067:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076400.724071:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076400.724075:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076400.724082:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076400.724099:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076400.724110:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076400/real 1769076400] req@ffff9b0352a1a300 x1855007617259136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076400.724124:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617259136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076400.724143:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076400.724146:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076400.724150:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076400.724156:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076400.724174:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076400.724184:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076400/real 1769076400] req@ffff9b0352a18e00 x1855007617259264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076400.724196:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617259264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076400.724206:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076400.724209:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076400.724213:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076400.724222:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076400.724307:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076400.724331:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076400/real 1769076400] req@ffff9b0352a18700 x1855007617259392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076400.724354:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617259392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076400.724365:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076400.724369:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076400.724374:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076401.235398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076401.619350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076401.619378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076401.619461:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076402.259370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076402.643366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076402.643408:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076402.643550:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076403.283341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076403.667340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076403.667341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076403.667412:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076404.307309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076404.691304:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076404.691317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076404.691404:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076405.331321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076405.715355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076405.715388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076405.715466:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076405.843377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076405.843390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076405.843396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076405.843408:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3395 00000100:00080000:3.0:1769076405.843420:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076405.843430:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076405.843482:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617259520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076405.843556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076405.843563:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076405.843567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076405.843575:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3395 00000100:00080000:3.0:1769076405.843582:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076405.843589:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076405.843604:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076405.843623:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617259648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076405.843639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076405.843645:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617259520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076405.843646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076405.843652:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076405.843682:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3395 00000100:00080000:3.0:1769076405.843687:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076405.843730:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076405.843739:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076405.843759:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617259776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076405.843775:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076405.843779:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076405.843782:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076405.843792:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3395 00000800:00000100:1.0:1769076405.843794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076405.843806:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076405.843815:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076405.843824:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076405.843842:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617259904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076405.843856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076405.843860:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076405/real 1769076405] req@ffff9b0352a18a80 x1855007617259520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076405.843863:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076405.843871:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076405.843893:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3395 00000100:00080000:3.0:1769076405.843904:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076405.843907:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617259520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076405.843922:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:1.0:1769076405.843930:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769076405.843939:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769076405.843940:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076405.843952:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617259520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076405.843979:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617260032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076405.843989:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076405.844041:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076405.844071:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076405/real 1769076405] req@ffff9b0352a1b480 x1855007617259648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076405.844097:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617259648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076405.844115:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076405.844122:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076405.844129:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076405.844142:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076405.844182:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076405.844203:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076405/real 1769076405] req@ffff9b0352a1ad80 x1855007617259776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076405.844219:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617259776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076405.844271:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076405.844277:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076405.844282:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076405.844290:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076405.844317:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076405.844331:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076405/real 1769076405] req@ffff9b0352a1b100 x1855007617259904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076405.844346:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617259904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076405.844356:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076405.844360:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076405.844364:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617259904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076405.844386:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076405.844406:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076405.844417:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076405/real 1769076405] req@ffff9b0352a1a680 x1855007617260032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076405.844431:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617260032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076405.844441:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076405.844445:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076405.844449:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076406.355366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076406.739357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076406.739372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076406.739426:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076407.379398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076407.763351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076407.763359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076407.763568:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076408.403359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076408.787365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076408.787379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076408.787552:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076409.427349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076409.811371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076409.811414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076409.811492:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076410.451375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076410.835370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076410.835386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076410.835489:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076410.963469:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076410.963480:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076410.963486:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076410.963498:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3400 00000100:00080000:1.0:1769076410.963512:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076410.963521:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076410.963569:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617260160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076410.963656:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076410.963665:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076410.963670:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076410.963685:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3400 00000100:00100000:3.0:1769076410.963686:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617260160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076410.963698:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076410.963711:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076410.963728:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076410.963759:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617260288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076410.963780:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076410.963788:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076410.963795:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076410.963799:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076410.963808:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3400 00000100:00080000:1.0:1769076410.963815:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076410.963821:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076410.963831:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076410.963842:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076410/real 1769076410] req@ffff9b0343ded180 x1855007617260160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076410.963853:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617260416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076410.963888:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076410.963892:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076410.963895:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076410.963904:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3400 00000100:00080000:1.0:1769076410.963908:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076410.963912:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076410.963918:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617260160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076410.963919:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076410.963942:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617260544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076410.963988:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076410.963989:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076410.964003:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076410.964007:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769076410.964009:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076410.964025:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617260160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076410.964026:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3400 00000100:00080000:1.0:1769076410.964037:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076410.964044:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076410.964050:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076410.964063:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617260672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076410.964072:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076410.964122:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076410.964148:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076410/real 1769076410] req@ffff9b0343dedf80 x1855007617260288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076410.964177:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617260288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076410.964195:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076410.964202:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076410.964210:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076410.964223:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076410.964298:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076410.964316:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076410/real 1769076410] req@ffff9b0343def100 x1855007617260416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076410.964332:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617260416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076410.964342:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076410.964347:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076410.964351:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076410.964360:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076410.964382:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076410.964393:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076410/real 1769076410] req@ffff9b0343dee300 x1855007617260544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076410.964408:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617260544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076410.964419:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076410.964422:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076410.964426:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076410.964448:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076410.964468:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076410.964479:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076410/real 1769076410] req@ffff9b0343dec380 x1855007617260672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076410.964493:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617260672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076410.964503:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076410.964506:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076410.964509:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076411.475382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076411.859362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076411.859399:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076411.859425:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076412.499352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076412.883370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076412.883439:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076412.883511:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076413.523378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076413.907404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076413.907440:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076413.907482:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076414.547382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076414.931365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076414.931384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076414.931466:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076415.571333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076415.955361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076415.955414:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076415.955509:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076416.083396:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076416.083412:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076416.083419:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076416.083440:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3405 00000100:00080000:1.0:1769076416.083458:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076416.083469:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076416.083580:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617260800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076416.083632:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076416.083639:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076416.083641:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076416.083648:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3405 00000100:00080000:1.0:1769076416.083652:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076416.083657:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076416.083672:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076416.083691:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617260928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076416.083706:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076416.083711:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076416.083716:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076416.083722:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617260800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076416.083727:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3405 00000100:00080000:1.0:1769076416.083739:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076416.083744:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076416.083751:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076416.083769:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617261056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076416.083781:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076416.083785:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076416.083789:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076416.083793:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3405 00000100:00080000:1.0:1769076416.083799:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076416.083804:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076416.083811:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076416.083825:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617261184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076416.083838:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076416.083842:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076416.083845:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769076416.083853:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076416.083856:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3405 00000100:00080000:1.0:1769076416.083865:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076416.083871:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076416.083880:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076416.083901:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617261312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076416.083945:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076416/real 1769076416] req@ffff9b0343dec700 x1855007617260800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076416.083975:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617260800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076416.083992:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076416.084016:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076416.084023:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617260800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076416.084060:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076416.084091:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076416.084108:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076416/real 1769076416] req@ffff9b0343dec000 x1855007617260928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076416.084124:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617260928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076416.084135:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076416.084140:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076416.084144:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617260928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076416.084152:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076416.084173:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076416.084192:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076416/real 1769076416] req@ffff9b0343dec380 x1855007617261056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076416.084231:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617261056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076416.084322:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076416.084328:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076416.084333:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076416.084342:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076416.084380:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076416.084396:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076416/real 1769076416] req@ffff9b0343dee300 x1855007617261184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076416.084412:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617261184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076416.084423:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076416.084427:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076416.084431:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076416.084438:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076416.084457:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076416.084468:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076416/real 1769076416] req@ffff9b0343def100 x1855007617261312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076416.084482:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617261312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076416.084500:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076416.084504:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076416.084508:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076416.595400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076416.979406:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076416.979426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076416.979535:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076417.619410:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076418.003369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076418.003375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076418.003473:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076418.643377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076419.027381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076419.027393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076419.027510:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076419.667367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076420.051359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076420.051388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076420.051482:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076420.691504:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076421.075371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076421.075412:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076421.075628:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076421.203426:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076421.203469:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076421.203476:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076421.203489:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3410 00000100:00080000:2.0:1769076421.203502:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076421.203515:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076421.203569:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617261440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076421.203632:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076421.203639:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076421.203643:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076421.203654:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3410 00000100:00080000:2.0:1769076421.203663:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076421.203669:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076421.203682:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076421.203695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617261440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076421.203709:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617261568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076421.203728:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076421.203732:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076421.203734:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076421.203739:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3410 00000100:00080000:2.0:1769076421.203744:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076421.203752:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076421.203762:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076421.203784:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617261696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076421.203805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076421.203805:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076421.203818:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076421.203825:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076421.203838:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3410 00000100:00080000:2.0:1769076421.203851:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076421.203861:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076421.203867:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076421/real 1769076421] req@ffff9b03482c5880 x1855007617261440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076421.203877:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076421.203890:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617261440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076421.203905:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769076421.203909:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617261824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076421.203917:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076421.203939:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617261440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076421.203950:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076421.203955:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076421.203959:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076421.203965:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3410 00000100:00080000:2.0:1769076421.203971:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076421.203976:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076421.203986:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076421.203987:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076421.204020:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617261952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076421.204045:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076421.204075:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076421/real 1769076421] req@ffff9b03482c6680 x1855007617261568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076421.204095:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617261568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076421.204111:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076421.204116:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076421.204123:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076421.204135:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076421.204164:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076421.204174:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076421/real 1769076421] req@ffff9b03482c5c00 x1855007617261696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076421.204182:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617261696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076421.204189:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076421.204191:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076421.204193:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076421.204199:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076421.204212:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076421.204218:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076421/real 1769076421] req@ffff9b03482c4e00 x1855007617261824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076421.204226:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617261824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076421.204262:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076421.204264:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076421.204267:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076421.204287:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076421.204304:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076421.204314:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076421/real 1769076421] req@ffff9b03482c7b80 x1855007617261952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076421.204322:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617261952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076421.204328:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076421.204331:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076421.204333:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617261952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076421.715340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076422.099379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076422.099490:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076422.099512:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076422.739375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076423.123368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076423.123372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076423.123515:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076423.763437:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076424.147423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076424.147425:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076424.147475:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076424.787409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076425.171386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076425.171402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076425.171484:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076425.811429:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076426.195345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076426.195401:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076426.195515:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076426.323460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076426.323478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076426.323487:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076426.323505:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3415 00000100:00080000:3.0:1769076426.323522:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076426.323534:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076426.323589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617262080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076426.323642:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076426.323651:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076426.323656:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076426.323665:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3415 00000100:00080000:3.0:1769076426.323673:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076426.323681:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076426.323697:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076426.323725:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617262208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076426.323751:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076426.323763:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1769076426.323764:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617262080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076426.323769:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076426.323786:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3415 00000100:00080000:3.0:1769076426.323795:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076426.323804:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076426.323817:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076426.323844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617262336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076426.323862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076426.323874:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076426.323884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076426.323890:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076426.323900:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3415 00000100:00080000:3.0:1769076426.323910:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076426.323921:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076426.323924:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076426/real 1769076426] req@ffff9b0352a18380 x1855007617262080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076426.323942:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076426.323973:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617262464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076426.323980:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617262080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076426.324030:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076426.324036:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076426.324038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076426.324051:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617262080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076426.324054:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076426.324063:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076426.324074:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3415 00000100:00080000:3.0:1769076426.324084:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076426.324089:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262208:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076426.324100:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076426.324115:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076426.324127:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076426.324148:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617262592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076426.324151:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076426/real 1769076426] req@ffff9b0352a1b800 x1855007617262208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076426.324213:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617262208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076426.324230:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076426.324279:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076426.324285:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076426.324294:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076426.324323:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076426.324341:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076426/real 1769076426] req@ffff9b0352a19500 x1855007617262336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076426.324387:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617262336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076426.324398:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076426.324402:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076426.324406:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076426.324414:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076426.324436:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076426.324448:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076426/real 1769076426] req@ffff9b0352a18000 x1855007617262464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076426.324462:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617262464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076426.324472:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076426.324475:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076426.324479:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076426.324498:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076426.324516:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076426.324527:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076426/real 1769076426] req@ffff9b0352a19c00 x1855007617262592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076426.324543:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617262592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076426.324572:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076426.324578:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076426.324585:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076426.835411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076427.219368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076427.219384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076427.219480:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076427.859375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076428.243457:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076428.243826:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076428.243944:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076428.883422:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769076429.267969:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076429.268061:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076429.268199:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076429.907398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076430.291499:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076430.291754:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076430.291825:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076430.931360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076431.315854:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076431.315944:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076431.316401:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076431.443442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076431.443458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076431.443466:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076431.443482:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3420 00000100:00080000:1.0:1769076431.443497:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076431.443509:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076431.443556:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617262720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076431.443598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076431.443605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076431.443609:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076431.443618:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3420 00000100:00080000:1.0:1769076431.443625:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076431.443634:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076431.443650:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076431.443668:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617262720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076431.443673:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617262848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076431.443692:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076431.443715:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076431.443720:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076431.443727:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3420 00000100:00080000:1.0:1769076431.443733:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076431.443739:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076431.443748:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769076431.443762:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076431.443766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617262976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076431.443782:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076431.443788:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076431.443791:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076431.443797:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3420 00000100:00080000:1.0:1769076431.443803:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076431.443809:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076431.443818:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076431.443821:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076431/real 1769076431] req@ffff9b0343dedf80 x1855007617262720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076431.443835:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617263104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076431.443855:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076431.443861:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076431.443864:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076431.443870:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617262720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076431.443871:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3420 00000100:00080000:1.0:1769076431.443878:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076431.443885:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:3.0:1769076431.443890:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769076431.443892:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1769076431.443898:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076431.443906:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617262720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076431.443908:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617263232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076431.443944:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076431.443988:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076431.444012:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076431/real 1769076431] req@ffff9b0343ded180 x1855007617262848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076431.444029:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617262848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076431.444041:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076431.444045:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076431.444049:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076431.444057:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076431.444082:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076431.444093:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076431/real 1769076431] req@ffff9b0343ded880 x1855007617262976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076431.444107:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617262976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076431.444125:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076431.444129:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076431.444133:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617262976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076431.444140:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076431.444157:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076431.444167:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076431/real 1769076431] req@ffff9b0343def800 x1855007617263104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076431.444181:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617263104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076431.444191:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076431.444194:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076431.444198:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076431.444204:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076431.444221:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076431.444288:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076431/real 1769076431] req@ffff9b0343deea00 x1855007617263232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076431.444308:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617263232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076431.444318:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076431.444322:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076431.444327:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076431.955407:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076432.339523:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076432.340074:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076432.340099:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076432.979360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076433.363425:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076433.363425:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076433.363520:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076434.003377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076434.387403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076434.387407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076434.387528:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076435.027344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076435.411381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076435.411384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076435.411524:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076436.051373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076436.435400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076436.435410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076436.435517:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076436.563492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076436.563509:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076436.563517:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076436.563533:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3425 00000100:00080000:2.0:1769076436.563551:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076436.563562:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076436.563627:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617263360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076436.563672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076436.563678:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076436.563681:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076436.563687:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3425 00000100:00080000:2.0:1769076436.563691:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076436.563696:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076436.563711:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076436.563729:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617263488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076436.563746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076436.563752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076436.563754:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076436.563763:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3425 00000100:00080000:2.0:1769076436.563777:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076436.563780:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617263360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076436.563789:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076436.563806:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076436.563830:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617263616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076436.563843:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076436.563847:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076436.563850:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076436.563856:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3425 00000100:00080000:2.0:1769076436.563860:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076436.563866:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076436.563873:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076436.563885:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617263744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076436.563896:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076436.563901:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076436.563904:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076436.563909:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3425 00000100:00080000:2.0:1769076436.563914:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076436.563919:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076436.563924:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769076436.563934:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076436.563951:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617263872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076436.564026:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076436/real 1769076436] req@ffff9b03482c5f80 x1855007617263360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076436.564064:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617263360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076436.564080:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076436.564086:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076436.564092:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617263360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076436.564126:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076436.564164:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076436.564212:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076436/real 1769076436] req@ffff9b03482c6a00 x1855007617263488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076436.564283:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617263488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076436.564304:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076436.564309:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076436.564314:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076436.564323:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076436.564354:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076436.564368:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076436/real 1769076436] req@ffff9b03482c4000 x1855007617263616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076436.564383:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617263616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076436.564394:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076436.564398:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076436.564402:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076436.564409:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076436.564428:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076436.564439:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076436/real 1769076436] req@ffff9b03482c7100 x1855007617263744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076436.564453:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617263744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076436.564462:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076436.564466:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076436.564471:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076436.564479:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076436.564497:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076436.564507:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076436/real 1769076436] req@ffff9b03482c4700 x1855007617263872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076436.564521:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617263872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076436.564535:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076436.564541:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076436.564548:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617263872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076437.075367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076437.459378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076437.459434:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076437.459554:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076438.099401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076438.483429:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076438.483433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076438.483462:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076439.123386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076439.507307:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076439.507309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076439.507363:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076440.147366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076440.531356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076440.531366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076440.531539:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076441.171320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076441.555362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076441.555375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076441.555477:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076441.683416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076441.683432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076441.683439:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076441.683455:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3430 00000100:00080000:3.0:1769076441.683472:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076441.683485:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076441.683537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617264000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076441.683585:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076441.683592:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076441.683595:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076441.683603:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3430 00000100:00080000:3.0:1769076441.683609:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076441.683616:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076441.683631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076441.683653:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617264128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076441.683682:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617264000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076441.683688:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076441.683699:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076441.683703:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076441.683713:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3430 00000100:00080000:3.0:1769076441.683722:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076441.683731:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076441.683742:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076441.683777:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617264256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076441.683785:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076441.683810:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076441.683818:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076441.683831:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076441.683843:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3430 00000100:00080000:3.0:1769076441.683853:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076441.683855:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076441/real 1769076441] req@ffff9b0352a19c00 x1855007617264000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076441.683871:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076441.683889:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076441.683920:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617264000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076441.683927:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617264384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076441.683956:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076441.683970:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076441.683977:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769076441.683984:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769076441.683988:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617264000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076441.683993:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076441.684008:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3430 00000100:00080000:3.0:1769076441.684018:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076441.684025:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264128:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076441.684034:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076441.684050:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076441.684066:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076441.684086:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617264512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076441.684128:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076441/real 1769076441] req@ffff9b0352a18000 x1855007617264128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076441.684190:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617264128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076441.684204:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076441.684213:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076441.684220:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076441.684289:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076441.684322:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076441.684342:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076441/real 1769076441] req@ffff9b0352a19500 x1855007617264256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076441.684366:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617264256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076441.684382:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076441.684388:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076441.684395:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076441.684407:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076441.684447:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076441.684469:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076441/real 1769076441] req@ffff9b0352a1b800 x1855007617264384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076441.684488:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617264384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076441.684498:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076441.684502:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076441.684506:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076441.684529:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076441.684553:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076441.684565:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076441/real 1769076441] req@ffff9b0352a18380 x1855007617264512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076441.684579:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617264512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076441.684589:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076441.684593:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076441.684597:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076442.195431:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076442.579334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076442.579356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076442.579448:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076443.219360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076443.603323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076443.603345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076443.603408:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076444.243378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076444.627381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076444.627400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076444.627505:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076445.267346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076445.651423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076445.651423:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076445.651424:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076446.291391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076446.675361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076446.675401:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076446.675513:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076446.803305:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076446.803324:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076446.803328:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076446.803340:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3435 00000100:00080000:3.0:1769076446.803366:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076446.803374:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076446.803413:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617264640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076446.803444:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076446.803454:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076446.803457:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076446.803462:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3435 00000100:00080000:3.0:1769076446.803467:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076446.803473:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076446.803484:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076446.803500:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617264640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076446.803500:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617264768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076446.803517:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076446.803520:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076446.803521:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076446.803526:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3435 00000100:00080000:3.0:1769076446.803529:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076446.803532:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076446.803536:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076446.803545:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617264896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076446.803553:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076446.803556:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076446.803557:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076446.803560:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3435 00000100:00080000:3.0:1769076446.803562:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076446.803565:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076446.803568:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076446.803597:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617265024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076446.803603:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076446.803605:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076446.803607:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076446.803610:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3435 00000100:00080000:3.0:1769076446.803612:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076446.803614:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076446.803617:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076446.803624:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617265152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076446.803647:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076446.803672:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076446.803691:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076446.803705:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076447.315384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076447.699366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076447.699417:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076447.699471:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076448.339330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076448.723372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076448.723433:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076448.723434:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076449.363337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076449.747379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076449.747471:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076449.747494:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769076449.875866:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769076449.876015:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769076449.876046:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076449.876127:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076449.876140:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076449.876169:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076449.876200:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076449.876223:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076446/real 1769076449] req@ffff9b0352a1b100 x1855007617264768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076449.876366:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617264768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076449.876389:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076449.876401:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076449.876411:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076449.876429:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076446/real 1769076449] req@ffff9b0352a1ad80 x1855007617264896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076449.876452:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617264896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076449.876467:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076449.876474:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076449.876481:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617264896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076449.876495:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076446/real 1769076449] req@ffff9b0352a1b480 x1855007617265024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076449.876524:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617265024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076449.876540:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076449.876548:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076449.876554:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076449.876569:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076446/real 1769076449] req@ffff9b0352a18a80 x1855007617265152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076449.876598:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617265152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076449.876614:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076449.876622:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076449.876631:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076449.876650:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076446/real 1769076449] req@ffff9b0352a1a680 x1855007617264640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076449.876686:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617264640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076449.876702:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076449.876709:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076449.876715:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617264640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076450.387320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076450.771326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076450.771330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076450.771372:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076451.411329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076451.795354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076451.795388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076451.795445:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076451.923372:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076451.923388:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076451.923395:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076451.923409:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3441 00000100:00080000:3.0:1769076451.923427:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076451.923437:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076451.923495:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617265280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076451.923545:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076451.923553:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076451.923556:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076451.923564:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3441 00000100:00080000:3.0:1769076451.923570:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076451.923577:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076451.923594:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076451.923614:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617265280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076451.923614:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617265408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076451.923633:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076451.923638:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076451.923641:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076451.923647:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3441 00000100:00080000:3.0:1769076451.923653:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076451.923660:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076451.923667:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076451.923680:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617265536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076451.923693:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076451.923699:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076451.923701:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076451.923706:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3441 00000100:00080000:3.0:1769076451.923712:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076451.923717:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769076451.923723:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769076451.923725:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076451.923738:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617265664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076451.923752:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076451.923756:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076451.923759:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076451.923765:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3441 00000100:00080000:3.0:1769076451.923769:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076451.923775:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076451.923781:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076451.923791:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076451/real 1769076451] req@ffff9b0352a18700 x1855007617265280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076451.923795:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617265792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076451.923844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617265280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076451.923873:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076451.923882:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076451.923906:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617265280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076451.923980:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076451.924012:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076451.924027:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076451/real 1769076451] req@ffff9b0352a18e00 x1855007617265408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076451.924041:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617265408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076451.924051:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076451.924055:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076451.924059:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076451.924065:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076451.924086:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076451.924096:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076451/real 1769076451] req@ffff9b0352a1a300 x1855007617265536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076451.924113:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617265536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076451.924127:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076451.924132:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076451.924137:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076451.924148:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076451.924182:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076451.924200:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076451/real 1769076451] req@ffff9b0352a19f80 x1855007617265664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076451.924217:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617265664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076451.924230:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076451.924287:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076451.924292:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076451.924301:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076451.924332:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076451.924348:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076451/real 1769076451] req@ffff9b0352a1aa00 x1855007617265792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076451.924387:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617265792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076451.924400:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076451.924405:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076451.924410:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617265792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076452.435295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076452.819309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076452.819392:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076452.819447:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076453.459287:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076453.843415:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076453.843415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076453.843455:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076454.483343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076454.867299:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076454.867344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076454.867402:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076455.507294:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076455.891360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076455.891372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076455.891432:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076456.531365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076456.915367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076456.915379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076456.915462:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076457.043536:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.043551:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076457.043558:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.043572:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3446 00000100:00080000:0.0:1769076457.043587:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.043613:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076457.043661:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617265920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.043704:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.043711:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076457.043714:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.043722:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3446 00000100:00080000:0.0:1769076457.043728:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.043735:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076457.043750:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076457.043770:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617266048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.043787:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076457.043797:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617265920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076457.043798:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076457.043807:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.043827:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3446 00000100:00080000:0.0:1769076457.043863:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.043871:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076457.043881:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076457.043899:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617266176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.043920:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.043928:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076457.043933:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.043940:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3446 00000100:00080000:0.0:1769076457.043948:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.043956:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076457.043980:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076457.044009:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617266304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076457.044033:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076457.044042:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076457.044047:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076457.044055:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3446 00000100:00080000:0.0:1769076457.044065:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076457.044076:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076457.044085:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769076457.044098:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076457.044108:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617266432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076457.044182:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9b0365a3bb80 x1855007617265920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076457.044213:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617265920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.044282:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.044291:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076457.044298:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617265920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076457.044354:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076457.044400:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076457.044424:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9b0365a3b100 x1855007617266048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076457.044448:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617266048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.044477:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.044486:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076457.044495:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076457.044516:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076457.044575:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076457.044610:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9b0365a39500 x1855007617266176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076457.044633:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617266176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.044652:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.044657:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076457.044661:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076457.044670:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076457.044698:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076457.044711:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9b0365a39180 x1855007617266304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076457.044727:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617266304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.044737:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.044740:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076457.044744:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076457.044752:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076457.044772:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076457.044790:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076457/real 1769076457] req@ffff9b0365a3ad80 x1855007617266432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076457.044813:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617266432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076457.044828:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076457.044859:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076457.044866:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076457.555325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076457.939388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076457.939388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076457.939460:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076458.579361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076458.963347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076458.963360:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076458.963363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076459.603347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076459.987318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076459.987318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076459.987382:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076460.627356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076461.011359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076461.011374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076461.011442:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076461.651374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076462.035364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076462.035368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076462.035508:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076462.163411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076462.163426:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076462.163432:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076462.163444:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3451 00000100:00080000:0.0:1769076462.163458:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076462.163467:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076462.163529:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617266560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076462.163569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076462.163574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076462.163576:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076462.163582:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3451 00000100:00080000:0.0:1769076462.163586:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076462.163590:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076462.163601:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076462.163616:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617266688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076462.163630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076462.163635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076462.163638:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076462.163644:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3451 00000100:00080000:0.0:1769076462.163647:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076462.163652:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076462.163658:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076462.163681:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617266560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076462.163684:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617266816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076462.163709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076462.163713:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076462.163716:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076462.163722:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3451 00000100:00080000:0.0:1769076462.163726:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076462.163730:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076462.163737:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076462.163813:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617266944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076462.163827:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076462.163835:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076462.163837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076462.163844:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076462.163852:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3451 00000100:00080000:0.0:1769076462.163859:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076462.163866:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076462.163884:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076462.163899:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617267072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076462.163913:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076462/real 1769076462] req@ffff9b0365a3ad80 x1855007617266560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076462.163943:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617266560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076462.163957:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076462.163964:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076462.163969:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617266560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076462.163980:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076462.164006:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076462.164020:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076462/real 1769076462] req@ffff9b0365a39180 x1855007617266688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076462.164036:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617266688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076462.164046:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076462.164050:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076462.164055:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076462.164085:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076462.164108:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076462.164119:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076462/real 1769076462] req@ffff9b0365a39500 x1855007617266816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076462.164134:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617266816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076462.164162:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076462.164169:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076462.164175:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076462.164187:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076462.164226:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076462.164297:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076462/real 1769076462] req@ffff9b0365a3b100 x1855007617266944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076462.164313:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617266944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076462.164324:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076462.164328:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076462.164332:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617266944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076462.164340:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076462.164367:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076462.164379:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076462/real 1769076462] req@ffff9b0365a3bb80 x1855007617267072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076462.164393:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617267072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076462.164403:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076462.164406:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076462.164411:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076462.675375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076463.059403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076463.059431:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076463.059555:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076463.699386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076464.083377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076464.083415:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076464.083433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076464.723380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076465.107355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076465.107366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076465.107447:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076465.747360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076466.131320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076466.131342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076466.131393:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076466.771341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076467.155329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076467.155331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076467.155390:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076467.283388:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076467.283405:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076467.283411:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076467.283423:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3456 00000100:00080000:0.0:1769076467.283436:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076467.283444:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076467.283491:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617267200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076467.283541:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076467.283547:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076467.283549:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076467.283555:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3456 00000100:00080000:0.0:1769076467.283559:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076467.283564:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076467.283576:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076467.283590:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617267328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076467.283602:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076467.283607:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076467.283609:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076467.283613:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3456 00000100:00080000:0.0:1769076467.283616:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076467.283620:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076467.283626:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076467.283639:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617267456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076467.283650:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076467.283654:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076467.283657:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076467.283691:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3456 00000100:00080000:0.0:1769076467.283697:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076467.283709:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076467.283712:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617267200:192.168.204.160@tcp:250:kworker.0 00000008:00100000:0.0:1769076467.283718:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076467.283737:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617267584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076467.283752:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076467.283756:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076467.283760:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076467.283765:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3456 00000100:00080000:0.0:1769076467.283771:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076467.283775:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076467.283781:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076467.283796:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617267712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076467.283811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076467.283865:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076467/real 1769076467] req@ffff9b0365a3aa00 x1855007617267200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076467.283890:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617267200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076467.283904:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076467.283910:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076467.283915:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617267200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076467.283925:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076467.283950:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076467.283964:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076467/real 1769076467] req@ffff9b0365a3b480 x1855007617267328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076467.283978:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617267328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076467.283989:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076467.283993:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076467.283998:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076467.284005:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076467.284023:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076467.284034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076467/real 1769076467] req@ffff9b0365a38e00 x1855007617267456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076467.284047:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617267456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076467.284057:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076467.284060:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076467.284064:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076467.284095:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076467.284114:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076467.284125:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076467/real 1769076467] req@ffff9b0365a39c00 x1855007617267584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076467.284139:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617267584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076467.284148:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076467.284153:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076467.284157:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076467.284164:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076467.284181:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076467.284192:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076467/real 1769076467] req@ffff9b0365a38a80 x1855007617267712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076467.284206:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617267712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076467.284215:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076467.284218:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076467.284222:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076467.795419:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076468.179382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076468.179433:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076468.179508:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076468.819344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076469.203376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076469.203376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076469.203449:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076469.843369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076470.227374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076470.227430:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076470.227471:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076470.867362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076471.251390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076471.251414:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076471.251495:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076471.891360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076472.275399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076472.275407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076472.275511:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076472.403407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076472.403416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076472.403420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076472.403429:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3461 00000100:00080000:1.0:1769076472.403437:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076472.403443:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076472.403476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617267840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076472.403501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076472.403504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076472.403505:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076472.403509:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3461 00000100:00080000:1.0:1769076472.403511:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076472.403513:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076472.403519:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076472.403528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617267968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076472.403535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076472.403538:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076472.403539:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076472.403542:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3461 00000100:00080000:1.0:1769076472.403543:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076472.403546:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076472.403549:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076472.403558:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617268096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076472.403560:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617267840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076472.403600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076472.403604:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076472.403607:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076472.403611:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3461 00000100:00080000:1.0:1769076472.403615:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076472.403618:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076472.403622:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076472.403632:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617268224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076472.403638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076472.403643:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076472.403644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076472.403649:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076472.403652:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3461 00000100:00080000:1.0:1769076472.403656:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076472.403659:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076472.403662:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076472.403670:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617268352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076472.403684:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076472/real 1769076472] req@ffff9b0343deed80 x1855007617267840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076472.403699:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617267840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076472.403707:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076472.403710:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076472.403713:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617267840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076472.403718:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076472.403735:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076472.403743:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076472/real 1769076472] req@ffff9b0343deca80 x1855007617267968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076472.403751:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617267968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076472.403757:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076472.403759:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076472.403761:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617267968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076472.403779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076472.403789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076472.403794:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076472/real 1769076472] req@ffff9b0343dece00 x1855007617268096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076472.403801:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617268096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076472.403806:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076472.403808:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076472.403810:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076472.403814:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076472.403823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076472.403829:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076472/real 1769076472] req@ffff9b0343defb80 x1855007617268224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076472.403841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617268224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076472.403845:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076472.403847:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076472.403849:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076472.403852:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076472.403861:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076472.403866:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076472/real 1769076472] req@ffff9b0343dee680 x1855007617268352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076472.403873:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617268352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076472.403878:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076472.403880:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076472.403882:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076472.915298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076473.299352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076473.299394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076473.299634:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076473.939369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076474.323380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076474.323387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076474.323435:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076474.963365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076475.347370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076475.347395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076475.347556:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076475.987371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076476.371407:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076476.371423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076476.371535:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076477.011380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076477.395386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076477.395438:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076477.395565:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076477.523347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076477.523358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076477.523362:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076477.523388:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3466 00000100:00080000:1.0:1769076477.523401:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076477.523410:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076477.523473:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617268480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076477.523548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076477.523552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076477.523554:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076477.523558:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3466 00000100:00080000:1.0:1769076477.523561:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076477.523565:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076477.523574:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076477.523586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617268608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076477.523597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076477.523600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076477.523601:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076477.523604:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3466 00000100:00080000:1.0:1769076477.523607:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076477.523609:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076477.523613:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076477.523628:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617268736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076477.523630:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617268480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076477.523646:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076477.523649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076477.523650:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076477.523653:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3466 00000100:00080000:1.0:1769076477.523655:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076477.523658:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076477.523663:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076477.523674:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617268864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076477.523682:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076477.523684:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076477.523686:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076477.523690:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3466 00000100:00080000:1.0:1769076477.523693:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076477.523695:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076477.523699:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076477.523709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617268992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076477.523719:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076477.523766:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076477/real 1769076477] req@ffff9b0343ded500 x1855007617268480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076477.523783:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617268480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076477.523793:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076477.523797:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076477.523800:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617268480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076477.523807:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076477.523826:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076477.523836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076477/real 1769076477] req@ffff9b0343def480 x1855007617268608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076477.523846:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617268608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076477.523853:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076477.523856:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076477.523858:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076477.523880:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076477.523894:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076477.523902:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076477/real 1769076477] req@ffff9b0343dee680 x1855007617268736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076477.523912:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617268736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076477.523925:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076477.523927:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076477.523930:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076477.523935:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076477.523947:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076477.523954:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076477/real 1769076477] req@ffff9b0343defb80 x1855007617268864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076477.523963:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617268864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076477.523970:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076477.523972:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076477.523974:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076477.523979:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076477.523990:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076477.523997:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076477/real 1769076477] req@ffff9b0343dece00 x1855007617268992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076477.524006:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617268992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076477.524012:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076477.524015:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076477.524018:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617268992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076478.035365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076478.419364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076478.419395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076478.419594:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076479.059380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076479.443405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076479.443514:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076479.443605:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076480.083396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076480.467372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076480.467417:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076480.467506:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076481.107366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076481.491378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076481.491413:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076481.491497:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076482.131368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769076482.515436:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076482.515436:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076482.515635:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076482.643381:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076482.643393:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076482.643400:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076482.643443:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3471 00000100:00080000:1.0:1769076482.643456:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076482.643465:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076482.643512:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617269120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076482.643551:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076482.643556:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076482.643559:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076482.643564:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3471 00000100:00080000:1.0:1769076482.643568:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076482.643573:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076482.643585:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076482.643599:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617269248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076482.643611:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076482.643616:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076482.643618:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076482.643624:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3471 00000100:00080000:1.0:1769076482.643628:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076482.643634:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076482.643640:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076482.643656:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617269376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076482.643674:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076482.643680:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076482.643683:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076482.643689:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3471 00000100:00080000:1.0:1769076482.643694:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076482.643707:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617269120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076482.643708:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076482.643735:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076482.643753:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617269504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076482.643768:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076482.643773:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076482.643775:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076482.643782:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3471 00000100:00080000:1.0:1769076482.643789:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076482.643799:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076482.643809:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076482.643832:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617269632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076482.643856:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076482.643928:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076482/real 1769076482] req@ffff9b0343deca80 x1855007617269120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076482.643955:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617269120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076482.643973:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076482.643980:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076482.643986:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617269120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076482.643997:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617269248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076482.644024:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076482.644039:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076482/real 1769076482] req@ffff9b0343deed80 x1855007617269248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076482.644054:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617269248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076482.644066:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076482.644070:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076482.644074:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617269248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076482.644113:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617269376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076482.644135:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076482.644147:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076482/real 1769076482] req@ffff9b0343deea00 x1855007617269376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076482.644161:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617269376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076482.644171:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076482.644175:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076482.644178:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617269376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076482.644186:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617269504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076482.644213:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076482.644295:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076482/real 1769076482] req@ffff9b0343def800 x1855007617269504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076482.644327:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617269504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076482.644345:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076482.644353:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076482.644360:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617269504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076482.644373:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617269632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076482.644443:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076482.644463:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076482/real 1769076482] req@ffff9b0343ded880 x1855007617269632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076482.644490:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617269632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076482.644501:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076482.644505:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076482.644509:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617269632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076483.155368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076483.539410:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076483.539496:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076483.539564:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076484.179443:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076484.563359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076484.563449:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076484.563552:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076485.203401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076485.587407:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076485.587411:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076485.587510:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076486.227387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076486.611409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076486.611421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076486.611496:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076487.251438:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076487.635429:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076487.635430:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076487.635613:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076487.763459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076487.763477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076487.763484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076487.763518:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3476 00000100:00080000:3.0:1769076487.763535:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076487.763546:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076487.763602:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617269760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076487.763644:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076487.763649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076487.763652:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076487.763657:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3476 00000100:00080000:3.0:1769076487.763661:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076487.763665:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076487.763678:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076487.763697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617269888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076487.763711:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076487.763716:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076487.763718:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076487.763724:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3476 00000100:00080000:3.0:1769076487.763727:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076487.763737:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076487.763751:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617269760:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769076487.763751:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076487.763776:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617270016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076487.763792:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076487.763798:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076487.763801:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076487.763807:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3476 00000100:00080000:3.0:1769076487.763812:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076487.763836:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076487.763843:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076487.763858:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617270144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076487.763872:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076487.763877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076487.763880:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076487.763890:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3476 00000800:00000100:2.0:1769076487.763903:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076487.763903:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076487.763919:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076487.763928:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076487.763947:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617270272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076487.763998:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076487/real 1769076487] req@ffff9b0352a1aa00 x1855007617269760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076487.764054:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617269760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076487.764069:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076487.764076:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076487.764082:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617269760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076487.764093:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617269888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076487.764129:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076487.764147:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076487/real 1769076487] req@ffff9b0352a19f80 x1855007617269888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076487.764162:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617269888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076487.764174:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076487.764178:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076487.764182:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617269888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076487.764214:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076487.764278:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076487.764297:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076487/real 1769076487] req@ffff9b0352a1a300 x1855007617270016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076487.764351:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617270016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076487.764369:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076487.764375:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076487.764383:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076487.764397:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076487.764453:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076487.764472:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076487/real 1769076487] req@ffff9b0352a18e00 x1855007617270144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076487.764488:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617270144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076487.764499:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076487.764513:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076487.764517:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076487.764524:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076487.764547:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076487.764558:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076487/real 1769076487] req@ffff9b0352a18700 x1855007617270272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076487.764571:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617270272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076487.764581:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076487.764584:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076487.764588:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076488.275379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076488.659381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076488.659387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076488.659562:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076489.299460:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076489.683396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076489.683402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076489.683561:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076490.323421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076490.707359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076490.707398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076490.707470:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076491.347387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076491.731391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076491.731489:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076491.731507:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076492.371404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076492.755399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076492.755400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076492.755495:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076492.883421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076492.883431:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076492.883436:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076492.883447:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3482 00000100:00080000:1.0:1769076492.883459:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076492.883467:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076492.883512:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617270400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076492.883562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076492.883569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076492.883573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076492.883582:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3482 00000100:00080000:1.0:1769076492.883591:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076492.883600:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076492.883613:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076492.883619:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617270400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076492.883637:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617270528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076492.883655:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076492.883659:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076492.883662:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076492.883668:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3482 00000100:00080000:1.0:1769076492.883673:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076492.883678:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076492.883683:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076492.883697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617270656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076492.883709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076492.883714:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076492.883716:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076492.883721:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3482 00000100:00080000:1.0:1769076492.883726:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076492.883731:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076492.883738:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076492.883750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617270784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076492.883761:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076492.883766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076492.883768:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076492.883772:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3482 00000100:00080000:1.0:1769076492.883778:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076492.883783:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076492.883788:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769076492.883800:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076492.883806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617270912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076492.883987:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076492/real 1769076492] req@ffff9b0343ded180 x1855007617270400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076492.884016:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617270400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076492.884077:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076492.884085:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076492.884091:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617270400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076492.884120:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076492.884148:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076492.884162:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076492/real 1769076492] req@ffff9b0343dedf80 x1855007617270528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076492.884176:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617270528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076492.884185:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076492.884188:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076492.884191:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076492.884197:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076492.884213:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076492.884222:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076492/real 1769076492] req@ffff9b0343def100 x1855007617270656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076492.884296:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617270656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076492.884313:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076492.884316:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076492.884320:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076492.884326:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076492.884348:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076492.884362:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076492/real 1769076492] req@ffff9b0343dee300 x1855007617270784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076492.884378:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617270784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076492.884387:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076492.884391:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076492.884394:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076492.884400:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076492.884414:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076492.884422:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076492/real 1769076492] req@ffff9b0343dec380 x1855007617270912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076492.884433:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617270912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076492.884440:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076492.884443:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076492.884447:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617270912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076493.395390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076493.779400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076493.779683:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076493.780295:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076494.419374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769076494.803392:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076494.803431:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076494.803493:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076495.443417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076495.827415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076495.827727:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076495.827757:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076496.467334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076496.851492:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076496.851517:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076496.851714:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076497.491308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076497.875491:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076497.875519:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076497.875558:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769076498.003363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.003376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076498.003381:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.003392:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3487 00000100:00080000:3.0:1769076498.003405:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.003413:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076498.003459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617271040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076498.003487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.003491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076498.003493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.003496:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3487 00000100:00080000:3.0:1769076498.003499:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.003501:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076498.003511:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076498.003534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617271168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076498.003542:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.003545:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076498.003547:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076498.003553:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617271040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076498.003553:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3487 00000100:00080000:3.0:1769076498.003558:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.003564:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076498.003570:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076498.003578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617271296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076498.003585:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.003599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076498.003600:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.003603:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3487 00000100:00080000:3.0:1769076498.003605:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.003607:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076498.003611:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076498.003619:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617271424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076498.003624:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076498.003626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076498.003627:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076498.003631:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3487 00000100:00080000:3.0:1769076498.003634:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076498.003636:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076498.003640:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076498.003647:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617271552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076498.003654:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076498.003702:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff9b0352a1a680 x1855007617271040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076498.003719:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617271040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076498.003727:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076498.003731:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076498.003734:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617271040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076498.003757:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076498.003773:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076498.003782:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff9b0352a18a80 x1855007617271168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076498.003790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617271168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076498.003796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076498.003799:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076498.003801:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076498.003804:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076498.003813:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076498.003819:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff9b0352a1b480 x1855007617271296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076498.003831:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617271296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076498.003836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076498.003838:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076498.003840:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076498.003853:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076498.003863:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076498.003868:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff9b0352a1ad80 x1855007617271424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076498.003875:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617271424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076498.003880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076498.003882:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076498.003884:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076498.003888:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076498.003896:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076498.003901:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076498/real 1769076498] req@ffff9b0352a1b100 x1855007617271552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076498.003908:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617271552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076498.003917:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076498.003919:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076498.003921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076498.515294:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076498.899371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076498.899374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076498.899417:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076499.539346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076499.923300:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076499.923304:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076499.923406:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076500.563308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076500.947344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076500.947344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076500.947419:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076501.587314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076501.971316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076501.971371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076501.971437:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076502.611305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076502.995334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076502.995334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076502.995403:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076503.123379:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076503.123391:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076503.123396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076503.123407:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3492 00000100:00080000:0.0:1769076503.123419:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076503.123426:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076503.123471:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617271680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076503.123504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076503.123507:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076503.123509:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076503.123512:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3492 00000100:00080000:0.0:1769076503.123515:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076503.123517:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076503.123526:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076503.123537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617271808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076503.123545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076503.123548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076503.123549:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076503.123552:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3492 00000100:00080000:0.0:1769076503.123554:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076503.123557:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076503.123561:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076503.123565:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617271680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076503.123571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617271936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076503.123580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076503.123582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076503.123583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076503.123586:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3492 00000100:00080000:0.0:1769076503.123589:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076503.123592:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076503.123595:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076503.123603:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617272064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076503.123609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076503.123612:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076503.123614:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076503.123616:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3492 00000100:00080000:0.0:1769076503.123619:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076503.123621:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076503.123623:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076503.123630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617272192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076503.123649:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076503.123697:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076503/real 1769076503] req@ffff9b0365a38a80 x1855007617271680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076503.123713:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617271680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076503.123722:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076503.123726:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076503.123729:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617271680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076503.123735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076503.123751:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076503.123759:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076503/real 1769076503] req@ffff9b0365a39c00 x1855007617271808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076503.123767:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617271808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076503.123774:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076503.123776:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076503.123778:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076503.123800:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076503.123810:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076503.123816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076503/real 1769076503] req@ffff9b0365a38e00 x1855007617271936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076503.123823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617271936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076503.123829:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076503.123831:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076503.123833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617271936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076503.123837:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076503.123847:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076503.123852:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076503/real 1769076503] req@ffff9b0365a3b480 x1855007617272064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076503.123859:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617272064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076503.123864:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076503.123865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076503.123868:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076503.123872:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076503.123881:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076503.123886:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076503/real 1769076503] req@ffff9b0365a3aa00 x1855007617272192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076503.123893:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617272192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076503.123899:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076503.123900:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076503.123902:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076503.635350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076504.019315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076504.019319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076504.019451:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076504.659358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076505.043323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076505.043341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076505.043404:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076505.683334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076506.067346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076506.067350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076506.067395:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076506.707346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076507.091321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076507.091352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076507.091391:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076507.731312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076508.115296:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076508.115317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076508.115359:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076508.243449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076508.243467:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076508.243475:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076508.243490:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3497 00000100:00080000:1.0:1769076508.243506:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076508.243517:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076508.243579:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617272320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076508.243642:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076508.243650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076508.243655:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076508.243664:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3497 00000100:00080000:1.0:1769076508.243673:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076508.243682:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076508.243700:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076508.243723:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617272320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076508.243734:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617272448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076508.243765:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076508.243773:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076508.243778:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076508.243788:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3497 00000100:00080000:1.0:1769076508.243798:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076508.243809:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076508.243821:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076508.243822:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076508.243859:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617272576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076508.243882:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076508/real 1769076508] req@ffff9b0343dec000 x1855007617272320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076508.243884:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076508.243904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076508.243915:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076508.243926:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617272320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076508.243930:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3497 00000100:00080000:1.0:1769076508.243949:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076508.243952:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076508.243966:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769076508.244007:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076508.244014:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617272320:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769076508.244027:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076508.244052:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076508.244064:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617272704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076508.244092:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076508.244093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076508.244106:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076508.244112:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076508.244119:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076508/real 1769076508] req@ffff9b0343dec700 x1855007617272448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076508.244128:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3497 00000100:00080000:1.0:1769076508.244146:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076508.244156:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617272448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076508.244159:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076508.244176:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769076508.244180:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076508.244190:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076508.244196:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076508.244205:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617272832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076508.244213:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076508.244306:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076508.244341:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076508/real 1769076508] req@ffff9b0343dec380 x1855007617272576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076508.244368:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617272576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076508.244385:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076508.244392:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076508.244400:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076508.244428:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076508.244488:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076508.244506:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076508/real 1769076508] req@ffff9b0343dee300 x1855007617272704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076508.244522:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617272704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076508.244532:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076508.244536:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076508.244540:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076508.244548:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076508.244567:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076508.244578:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076508/real 1769076508] req@ffff9b0343def100 x1855007617272832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076508.244593:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617272832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076508.244602:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076508.244605:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076508.244609:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617272832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076508.755364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076509.139310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076509.139319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076509.139405:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076509.779308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076510.163358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076510.163358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076510.163430:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076510.803393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076511.187317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076511.187324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076511.187380:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076511.827310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076512.211385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076512.211442:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076512.211522:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076512.851342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076513.235336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076513.235336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076513.235412:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076513.363355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076513.363364:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076513.363368:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076513.363376:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3502 00000100:00080000:2.0:1769076513.363384:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076513.363389:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076513.363420:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617272960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076513.363442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076513.363445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076513.363447:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076513.363450:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3502 00000100:00080000:2.0:1769076513.363452:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076513.363455:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076513.363463:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076513.363473:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617273088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076513.363476:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617272960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076513.363482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076513.363486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076513.363487:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076513.363490:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3502 00000100:00080000:2.0:1769076513.363491:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076513.363494:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076513.363497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076513.363506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617273216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076513.363516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076513.363519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076513.363520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076513.363523:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3502 00000100:00080000:2.0:1769076513.363532:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076513.363535:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076513.363538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076513.363546:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617273344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076513.363552:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076513.363555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076513.363556:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076513.363559:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3502 00000100:00080000:2.0:1769076513.363561:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076513.363564:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076513.363568:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076513.363576:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617273472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076513.363587:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076513.363613:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076513.363627:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076513.363649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076513.875278:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076514.259337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076514.259365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076514.259426:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076514.899289:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076515.283309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076515.283330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076515.283429:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076515.923280:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076516.307347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076516.307347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076516.307471:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769076516.435459:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769076516.435494:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769076516.435508:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076516.435540:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076516.435549:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076516.435557:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769076516.435565:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:3.0:1769076516.435582:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076513/real 1769076516] req@ffff9b03482c6d80 x1855007617273088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076516.435609:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617273088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076516.435622:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076516.435626:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076516.435631:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076516.435641:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076513/real 1769076516] req@ffff9b03482c6300 x1855007617273216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076516.435654:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617273216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076516.435663:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076516.435667:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076516.435671:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076516.435677:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076513/real 1769076516] req@ffff9b03482c4a80 x1855007617273344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076516.435689:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617273344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076516.435697:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076516.435702:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076516.435705:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076516.435712:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076513/real 1769076516] req@ffff9b03482c5180 x1855007617272960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076516.435725:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617272960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076516.435733:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076516.435737:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076516.435741:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617272960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076516.435768:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076513/real 1769076516] req@ffff9b03482c4380 x1855007617273472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076516.435787:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617273472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076516.435795:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076516.435799:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076516.435803:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076516.947287:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076517.331310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076517.331322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076517.331358:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076517.971312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076518.355352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076518.355360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076518.355466:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076518.483381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076518.483394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076518.483401:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076518.483414:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3507 00000100:00080000:2.0:1769076518.483427:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076518.483437:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076518.483479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617273600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076518.483518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076518.483523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076518.483525:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076518.483531:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3507 00000100:00080000:2.0:1769076518.483535:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076518.483539:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076518.483549:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076518.483564:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617273728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076518.483577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076518.483581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076518.483583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076518.483587:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3507 00000100:00080000:2.0:1769076518.483591:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076518.483595:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076518.483601:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076518.483611:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617273856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076518.483623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076518.483627:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076518.483630:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076518.483636:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3507 00000100:00080000:2.0:1769076518.483641:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076518.483646:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076518.483653:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076518.483669:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617273984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076518.483673:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617273600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076518.483691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076518.483700:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076518.483703:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076518.483711:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3507 00000100:00080000:2.0:1769076518.483719:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076518.483726:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076518.483736:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076518.483761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617274112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076518.483794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076518.483909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076518/real 1769076518] req@ffff9b03482c5500 x1855007617273600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076518.483944:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617273600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076518.483964:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076518.483973:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076518.483982:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617273600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076518.483997:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076518.484045:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076518.484072:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076518/real 1769076518] req@ffff9b03482c4380 x1855007617273728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076518.484101:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617273728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076518.484119:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076518.484125:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076518.484131:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076518.484143:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076518.484184:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076518.484208:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076518/real 1769076518] req@ffff9b03482c5180 x1855007617273856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076518.484279:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617273856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076518.484298:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076518.484305:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076518.484312:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076518.484362:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076518.484404:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076518.484428:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076518/real 1769076518] req@ffff9b03482c4a80 x1855007617273984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076518.484587:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617273984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076518.484623:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076518.484631:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076518.484639:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617273984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076518.484652:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617274112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076518.484699:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076518.484724:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076518/real 1769076518] req@ffff9b03482c6300 x1855007617274112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076518.484749:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617274112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076518.484766:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076518.484774:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076518.484781:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617274112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076518.995333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076519.379372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076519.379373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076519.379468:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076520.019369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076520.403398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076520.403398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076520.403551:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076521.043300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076521.427301:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076521.427301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076521.427409:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076522.067342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076522.451356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076522.451363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076522.451504:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076523.091338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076523.475352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076523.475394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076523.475551:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076523.603386:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076523.603407:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076523.603414:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076523.603431:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3512 00000100:00080000:0.0:1769076523.603447:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076523.603459:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076523.603524:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617274240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076523.603584:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076523.603593:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076523.603597:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076523.603608:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3512 00000100:00080000:0.0:1769076523.603618:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076523.603644:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076523.603662:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076523.603670:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617274240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076523.603693:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617274368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076523.603752:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076523.603758:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076523.603761:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076523.603766:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3512 00000100:00080000:0.0:1769076523.603770:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076523.603775:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076523.603782:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076523.603796:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617274496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076523.603807:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076523.603812:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076523.603815:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769076523.603819:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076523.603826:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3512 00000100:00080000:0.0:1769076523.603835:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076523.603841:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076523.603852:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076523.603871:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617274624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076523.603883:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076523/real 1769076523] req@ffff9b0365a3bb80 x1855007617274240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076523.603889:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076523.603900:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076523.603904:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076523.603911:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3512 00000100:00080000:0.0:1769076523.603916:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076523.603921:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617274240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076523.603925:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076523.603937:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1769076523.603942:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076523.603950:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076523.603955:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617274752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076523.603959:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617274240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076523.604000:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617274368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076523.604029:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076523.604047:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076523/real 1769076523] req@ffff9b0365a3b100 x1855007617274368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076523.604066:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617274368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076523.604093:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076523.604098:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076523.604103:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617274368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076523.604110:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617274496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076523.604133:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076523.604144:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076523/real 1769076523] req@ffff9b0365a39500 x1855007617274496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076523.604156:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617274496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076523.604170:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076523.604174:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076523.604177:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617274496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076523.604182:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617274624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076523.604196:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076523.604204:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076523/real 1769076523] req@ffff9b0365a39180 x1855007617274624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076523.604213:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617274624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076523.604221:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076523.604223:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076523.604226:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617274624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076523.604230:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617274752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076523.604289:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076523.604305:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076523/real 1769076523] req@ffff9b0365a3ad80 x1855007617274752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076523.604322:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617274752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076523.604333:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076523.604338:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076523.604343:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617274752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076524.115411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076524.499360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076524.499374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076524.499586:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076525.139366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076525.523321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076525.523321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076525.523416:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076526.163423:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076526.547411:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076526.547413:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076526.547519:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076527.187397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076527.571309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076527.571320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076527.571383:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076528.211323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076528.595363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076528.595394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076528.595484:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076528.723406:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076528.723420:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076528.723426:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076528.723438:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3517 00000100:00080000:3.0:1769076528.723450:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076528.723460:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076528.723507:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617274880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076528.723544:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076528.723551:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076528.723554:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076528.723560:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3517 00000100:00080000:3.0:1769076528.723564:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076528.723570:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076528.723581:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076528.723601:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617275008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076528.723626:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076528.723659:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076528.723662:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076528.723674:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3517 00000100:00100000:0.0:1769076528.723676:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617274880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076528.723679:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076528.723693:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076528.723702:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076528.723723:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617275136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076528.723737:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076528.723742:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076528.723745:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076528.723750:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3517 00000100:00080000:3.0:1769076528.723755:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076528.723762:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076528.723771:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769076528.723778:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076528.723837:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076528/real 1769076528] req@ffff9b0352a18380 x1855007617274880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076528.723871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617274880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076528.723889:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617275264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076528.723890:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076528.723908:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076528.723917:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617274880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076528.723930:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076528.723974:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076528.724001:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076528/real 1769076528] req@ffff9b0352a1b800 x1855007617275008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076528.724050:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617275008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076528.724070:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076528.724079:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076528.724086:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275008:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076528.724111:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076528.724117:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076528.724121:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076528.724128:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275136:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076528.724132:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3517 00000100:00080000:3.0:1769076528.724146:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076528.724156:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769076528.724181:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076528.724213:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076528/real 1769076528] req@ffff9b0352a19500 x1855007617275136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076528.724230:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076528.724286:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617275136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076528.724294:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617275392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076528.724329:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076528.724342:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076528.724351:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076528.724361:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076528.724389:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076528.724405:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076528/real 1769076528] req@ffff9b0352a18000 x1855007617275264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076528.724429:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617275264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076528.724439:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076528.724443:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076528.724447:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076528.724463:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076528.724482:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076528.724493:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076528/real 1769076528] req@ffff9b0352a19c00 x1855007617275392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076528.724507:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617275392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076528.724517:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076528.724520:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076528.724524:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076529.235407:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076529.619364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076529.619408:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076529.619463:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076530.259390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076530.643376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076530.643386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076530.643490:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076531.283328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076531.667415:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076531.667467:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076531.667562:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076532.307400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076532.691318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076532.691319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076532.691421:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076533.331385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076533.715312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076533.715318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076533.715405:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076533.843430:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076533.843444:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076533.843450:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076533.843463:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3523 00000100:00080000:3.0:1769076533.843477:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076533.843486:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076533.843538:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617275520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076533.843615:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076533.843620:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076533.843623:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076533.843628:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3523 00000100:00080000:3.0:1769076533.843660:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076533.843669:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076533.843686:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076533.843727:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617275648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076533.843742:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617275520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076533.843779:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076533.843785:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076533.843789:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076533.843798:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3523 00000100:00080000:3.0:1769076533.843808:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076533.843826:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076533.843834:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076533.843855:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617275776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076533.843868:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076533.843872:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076533.843874:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076533.843879:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3523 00000100:00080000:3.0:1769076533.843883:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076533.843888:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076533.843895:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769076533.843904:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076533.843910:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617275904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076533.843933:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076533.843937:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076533.843941:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076533.843948:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3523 00000100:00080000:3.0:1769076533.843952:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076533.843958:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076533.843964:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076533.844012:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076533/real 1769076533] req@ffff9b0352a19c00 x1855007617275520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076533.844013:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617276032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076533.844110:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617275520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076533.844123:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076533.844128:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076533.844132:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617275520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076533.844160:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076533.844184:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076533.844196:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076533/real 1769076533] req@ffff9b0352a18000 x1855007617275648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076533.844207:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617275648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076533.844214:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076533.844217:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076533.844220:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076533.844225:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076533.844275:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076533.844300:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076533/real 1769076533] req@ffff9b0352a19500 x1855007617275776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076533.844309:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617275776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076533.844315:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076533.844318:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076533.844320:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076533.844326:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076533.844340:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076533.844365:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076533/real 1769076533] req@ffff9b0352a1b800 x1855007617275904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076533.844385:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617275904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076533.844392:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076533.844395:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076533.844397:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617275904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076533.844402:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076533.844437:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076533.844450:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076533/real 1769076533] req@ffff9b0352a18380 x1855007617276032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076533.844465:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617276032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076533.844474:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076533.844478:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076533.844497:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076534.355320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076534.739316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076534.739316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076534.739358:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076535.379305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076535.763321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076535.763322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076535.763400:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076536.403337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076536.787373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076536.787381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076536.787574:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076537.427437:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076537.811332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076537.811332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076537.811395:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076538.451368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076538.835384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076538.835384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076538.835408:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076538.963328:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076538.963342:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076538.963349:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076538.963362:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3528 00000100:00080000:3.0:1769076538.963374:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076538.963400:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076538.963446:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617276160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076538.963525:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076538.963529:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076538.963530:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076538.963533:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3528 00000100:00080000:3.0:1769076538.963536:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076538.963539:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076538.963546:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076538.963556:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617276288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076538.963564:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076538.963567:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076538.963570:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076538.963572:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617276160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076538.963575:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3528 00000100:00080000:3.0:1769076538.963582:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076538.963584:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076538.963587:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076538.963594:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617276416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076538.963600:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076538.963602:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076538.963604:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076538.963607:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3528 00000100:00080000:3.0:1769076538.963610:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076538.963612:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076538.963616:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076538.963622:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617276544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076538.963629:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076538.963631:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076538.963632:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076538.963636:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3528 00000800:00000100:0.0:1769076538.963642:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076538.963643:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076538.963650:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076538.963656:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076538.963670:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617276672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076538.963699:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076538/real 1769076538] req@ffff9b0352a1b100 x1855007617276160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076538.963715:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617276160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076538.963723:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076538.963727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076538.963730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617276160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076538.963749:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076538.963767:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076538.963776:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076538/real 1769076538] req@ffff9b0352a1ad80 x1855007617276288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076538.963784:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617276288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076538.963790:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076538.963792:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076538.963794:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076538.963798:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076538.963808:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076538.963814:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076538/real 1769076538] req@ffff9b0352a1b480 x1855007617276416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076538.963821:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617276416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076538.963831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076538.963833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076538.963843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076538.963850:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076538.963870:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076538.963881:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076538/real 1769076538] req@ffff9b0352a18a80 x1855007617276544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076538.963895:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617276544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076538.963903:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076538.963907:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076538.963910:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076538.963914:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076538.963929:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076538.963936:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076538/real 1769076538] req@ffff9b0352a1a680 x1855007617276672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076538.963944:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617276672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076538.963949:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076538.963951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076538.963954:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076539.475396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076539.859408:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076539.859412:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076539.859556:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076540.499414:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076540.883378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076540.883391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076540.883495:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076541.523398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076541.907375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076541.907380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076541.907426:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076542.547345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076542.931335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076542.931364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076542.931473:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076543.571413:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076543.955425:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076543.955443:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076543.955484:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076544.083512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076544.083527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076544.083535:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076544.083550:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3533 00000100:00080000:1.0:1769076544.083563:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076544.083572:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076544.083619:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617276800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076544.083659:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076544.083664:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076544.083667:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076544.083672:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3533 00000100:00080000:1.0:1769076544.083676:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076544.083681:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076544.083692:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076544.083706:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617276928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076544.083719:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076544.083724:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076544.083726:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076544.083731:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3533 00000100:00080000:1.0:1769076544.083737:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076544.083742:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076544.083749:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076544.083762:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617277056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076544.083774:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617276800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076544.083788:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076544.083801:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076544.083803:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076544.083809:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3533 00000100:00080000:1.0:1769076544.083814:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076544.083818:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076544.083826:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076544.083843:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617277184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076544.083855:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076544.083860:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076544.083862:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076544.083866:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3533 00000100:00080000:1.0:1769076544.083872:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076544.083875:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076544.083882:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769076544.083890:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076544.083906:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617277312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076544.083956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076544/real 1769076544] req@ffff9b0343dedf80 x1855007617276800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076544.083982:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617276800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076544.084015:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076544.084023:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076544.084030:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617276800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076544.084046:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076544.084090:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076544.084125:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076544/real 1769076544] req@ffff9b0343ded180 x1855007617276928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076544.084151:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617276928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076544.084167:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076544.084173:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076544.084180:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617276928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076544.084276:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076544.084326:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076544.084349:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076544/real 1769076544] req@ffff9b0343ded880 x1855007617277056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076544.084409:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617277056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076544.084426:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076544.084433:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076544.084440:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076544.084453:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076544.084490:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076544.084506:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076544/real 1769076544] req@ffff9b0343def800 x1855007617277184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076544.084522:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617277184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076544.084532:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076544.084536:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076544.084540:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076544.084547:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076544.084566:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076544.084577:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076544/real 1769076544] req@ffff9b0343deea00 x1855007617277312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076544.084600:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617277312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076544.084609:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076544.084613:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076544.084616:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076544.595460:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076544.979296:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076544.979299:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076544.979387:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076545.619370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076546.003323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076546.003323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076546.003401:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076546.643314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076547.027306:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076547.027336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076547.027543:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076547.667362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076548.051319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076548.051320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076548.051399:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076548.691363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076549.075308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076549.075313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076549.075433:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076549.203379:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076549.203392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076549.203397:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076549.203409:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3538 00000100:00080000:3.0:1769076549.203422:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076549.203429:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076549.203474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617277440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076549.203504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076549.203507:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076549.203509:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076549.203512:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3538 00000100:00080000:3.0:1769076549.203514:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076549.203517:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076549.203525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076549.203535:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617277568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076549.203543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076549.203546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076549.203547:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076549.203551:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3538 00000100:00080000:3.0:1769076549.203554:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076549.203558:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076549.203562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076549.203574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617277696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076549.203577:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617277440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076549.203598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076549.203601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076549.203602:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076549.203606:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3538 00000100:00080000:3.0:1769076549.203608:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076549.203612:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076549.203619:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076549.203630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617277824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076549.203636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076549.203638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076549.203640:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076549.203644:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3538 00000100:00080000:3.0:1769076549.203647:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076549.203649:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076549.203653:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076549.203668:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617277952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076549.203677:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076549.203730:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076549/real 1769076549] req@ffff9b0352a18700 x1855007617277440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076549.203755:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617277440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076549.203771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076549.203777:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076549.203783:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617277440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076549.203795:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076549.203828:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076549.203846:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076549/real 1769076549] req@ffff9b0352a18e00 x1855007617277568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076549.203863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617277568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076549.203883:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076549.203889:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076549.203894:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076549.203926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076549.203972:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076549.203988:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076549/real 1769076549] req@ffff9b0352a1a300 x1855007617277696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076549.204020:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617277696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076549.204033:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076549.204038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076549.204043:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076549.204053:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076549.204080:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076549.204096:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076549/real 1769076549] req@ffff9b0352a19f80 x1855007617277824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076549.204128:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617277824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076549.204139:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076549.204144:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076549.204149:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076549.204157:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076549.204197:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076549.204212:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076549/real 1769076549] req@ffff9b0352a1aa00 x1855007617277952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076549.204229:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617277952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076549.204239:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076549.204244:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076549.204249:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617277952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076549.715328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076550.099364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076550.099377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076550.099443:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076550.739378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076551.123394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076551.123395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076551.123457:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076551.763391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076552.147314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076552.147320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076552.147387:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076552.787312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076553.171341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076553.171341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076553.171399:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076553.811331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076554.195327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076554.195331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076554.195360:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076554.323356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076554.323367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076554.323372:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076554.323383:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3543 00000100:00080000:3.0:1769076554.323392:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076554.323400:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076554.323438:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617278080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076554.323464:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076554.323469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076554.323471:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076554.323475:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3543 00000100:00080000:3.0:1769076554.323479:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076554.323483:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076554.323492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076554.323503:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617278208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076554.323513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076554.323517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076554.323519:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076554.323525:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3543 00000100:00080000:3.0:1769076554.323528:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076554.323532:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076554.323538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076554.323547:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617278080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076554.323550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617278336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076554.323570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076554.323573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076554.323575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076554.323579:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3543 00000100:00080000:3.0:1769076554.323583:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076554.323588:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076554.323594:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076554.323606:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617278464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076554.323616:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076554.323620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076554.323625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076554.323627:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076554.323634:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3543 00000100:00080000:3.0:1769076554.323639:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076554.323643:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076554.323649:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076554.323656:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076554/real 1769076554] req@ffff9b0352a1aa00 x1855007617278080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076554.323665:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617278592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076554.323675:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617278080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076554.323684:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076554.323688:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076554.323691:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617278080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076554.323698:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076554.323715:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076554.323723:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076554/real 1769076554] req@ffff9b0352a19f80 x1855007617278208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076554.323731:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617278208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076554.323736:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076554.323738:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076554.323740:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076554.323757:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076554.323767:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076554.323773:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076554/real 1769076554] req@ffff9b0352a1a300 x1855007617278336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076554.323780:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617278336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076554.323785:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076554.323787:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076554.323789:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076554.323792:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076554.323802:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076554.323807:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076554/real 1769076554] req@ffff9b0352a18e00 x1855007617278464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076554.323818:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617278464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076554.323823:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076554.323825:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076554.323827:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076554.323830:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076554.323838:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076554.323843:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076554/real 1769076554] req@ffff9b0352a18700 x1855007617278592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076554.323850:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617278592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076554.323855:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076554.323857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076554.323859:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076554.835326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076555.219379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076555.219380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076555.219450:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076555.859331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076556.243329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076556.243329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076556.243416:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076556.883334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076557.267302:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076557.267334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076557.267378:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076557.907367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076558.291321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076558.291370:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076558.291387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076558.931313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076559.315333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076559.315334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076559.315364:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076559.443356:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076559.443366:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076559.443371:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076559.443382:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3548 00000100:00080000:2.0:1769076559.443391:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076559.443398:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076559.443433:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617278720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076559.443461:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076559.443466:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076559.443468:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076559.443472:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3548 00000100:00080000:2.0:1769076559.443476:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076559.443481:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076559.443490:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076559.443504:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617278848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076559.443510:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617278720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076559.443526:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076559.443531:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076559.443533:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076559.443539:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3548 00000100:00080000:2.0:1769076559.443543:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076559.443548:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076559.443553:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076559.443565:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617278976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076559.443575:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076559.443581:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076559.443582:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076559.443587:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076559.443593:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3548 00000100:00080000:2.0:1769076559.443598:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076559.443603:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076559.443609:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076559.443623:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617279104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076559.443633:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076559/real 1769076559] req@ffff9b03482c6d80 x1855007617278720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076559.443635:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076559.443643:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076559.443648:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076559.443656:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3548 00000100:00080000:2.0:1769076559.443661:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076559.443665:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617278720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076559.443680:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076559.443680:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:0.0:1769076559.443690:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000008:00100000:2.0:1769076559.443690:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076559.443698:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617278720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076559.443707:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617279232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076559.443725:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076559.443752:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076559.443766:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076559/real 1769076559] req@ffff9b03482c4700 x1855007617278848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076559.443779:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617278848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076559.443788:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076559.443791:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076559.443795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076559.443801:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076559.443819:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076559.443828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076559/real 1769076559] req@ffff9b03482c7100 x1855007617278976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076559.443841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617278976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076559.443850:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076559.443854:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076559.443857:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617278976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076559.443863:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076559.443882:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076559.443892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076559/real 1769076559] req@ffff9b03482c4000 x1855007617279104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076559.443904:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617279104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076559.443912:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076559.443915:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076559.443919:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076559.443925:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076559.443945:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076559.443954:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076559/real 1769076559] req@ffff9b03482c6a00 x1855007617279232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076559.443966:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617279232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076559.443975:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076559.443978:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076559.443981:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076559.955340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076560.339605:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076560.339648:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076560.339921:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076560.979314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076561.363362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076561.363727:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076561.363734:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076562.003341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076562.387338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076562.387369:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076562.387388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076563.027325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076563.411363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076563.411610:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076563.411621:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076564.051335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076564.435326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076564.435356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076564.435396:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076564.563352:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076564.563360:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076564.563364:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076564.563373:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3553 00000100:00080000:3.0:1769076564.563381:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076564.563388:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076564.563417:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617279360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076564.563439:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076564.563443:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076564.563444:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076564.563448:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3553 00000100:00080000:3.0:1769076564.563451:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076564.563454:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076564.563463:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076564.563477:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617279360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076564.563477:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617279488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076564.563493:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076564.563495:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076564.563498:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076564.563502:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3553 00000100:00080000:3.0:1769076564.563504:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076564.563507:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076564.563511:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076564.563520:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617279616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076564.563526:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076564.563529:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076564.563530:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076564.563532:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3553 00000100:00080000:3.0:1769076564.563535:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076564.563538:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769076564.563543:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769076564.563545:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076564.563576:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076564/real 1769076564] req@ffff9b0352a1a680 x1855007617279360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076564.563591:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617279360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076564.563598:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076564.563602:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076564.563605:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617279360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076564.563623:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076564.563636:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617279744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076564.563642:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076564.563654:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076564/real 1769076564] req@ffff9b0352a18a80 x1855007617279488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076564.563656:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076564.563667:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769076564.563672:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617279488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076564.563684:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076564.563687:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076564.563689:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076564.563692:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076564.563703:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076564.563709:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076564/real 1769076564] req@ffff9b0352a1b480 x1855007617279616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076564.563716:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617279616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076564.563721:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076564.563723:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076564.563725:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076564.563733:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076564.563745:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076564.563752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076564/real 1769076564] req@ffff9b0352a1ad80 x1855007617279744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076564.563761:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617279744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076564.563766:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076564.563768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076564.563771:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279744:192.168.204.160@tcp:8:kworker.0 00000100:00080000:3.0:1769076564.563774:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076564.563785:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3553 00000100:00080000:3.0:1769076564.563789:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076564.563792:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076564.563797:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076564.563808:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617279872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076564.563901:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076564.563921:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076564.563932:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076564/real 1769076564] req@ffff9b0352a1b100 x1855007617279872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076564.563940:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617279872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076564.563945:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076564.563947:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076564.563949:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617279872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076565.075353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076565.459363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076565.459394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076565.459471:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076566.099337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076566.483355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076566.483371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076566.483408:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076567.123343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076567.507338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076567.507357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076567.507419:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076568.147331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076568.531308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076568.531309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076568.531388:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076569.171359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076569.555356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076569.555381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076569.555542:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076569.683400:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076569.683414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076569.683421:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076569.683434:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3558 00000100:00080000:2.0:1769076569.683446:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076569.683466:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076569.683511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617280000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076569.683546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076569.683551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076569.683554:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076569.683559:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3558 00000100:00080000:2.0:1769076569.683563:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076569.683567:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076569.683580:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076569.683612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617280128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076569.683626:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076569.683630:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076569.683633:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076569.683637:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3558 00000100:00080000:2.0:1769076569.683641:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076569.683649:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076569.683652:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617280000:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769076569.683655:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076569.683672:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617280256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076569.683702:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076569.683706:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076569.683709:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076569.683714:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3558 00000100:00080000:2.0:1769076569.683718:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076569.683723:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076569.683729:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076569.683743:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617280384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076569.683754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076569.683758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076569.683761:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076569.683766:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3558 00000100:00080000:2.0:1769076569.683772:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076569.683789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076569.683794:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076569.683803:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076569.683819:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617280512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076569.683872:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076569/real 1769076569] req@ffff9b03482c5f80 x1855007617280000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076569.683897:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617280000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076569.683910:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076569.683917:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076569.683922:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617280000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076569.683931:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076569.684010:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076569.684029:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076569/real 1769076569] req@ffff9b03482c7b80 x1855007617280128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076569.684046:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617280128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076569.684057:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076569.684061:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076569.684066:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076569.684097:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076569.684139:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076569.684150:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076569/real 1769076569] req@ffff9b03482c4e00 x1855007617280256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076569.684164:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617280256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076569.684173:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076569.684177:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076569.684181:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076569.684187:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076569.684205:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076569.684233:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076569/real 1769076569] req@ffff9b03482c5c00 x1855007617280384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076569.684248:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617280384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076569.684258:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076569.684261:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076569.684265:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076569.684271:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076569.684353:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076569.684368:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076569/real 1769076569] req@ffff9b03482c6680 x1855007617280512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076569.684383:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617280512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076569.684392:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076569.684396:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076569.684400:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076570.195386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076570.579335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076570.579362:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076570.579441:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076571.219385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076571.603370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076571.603377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076571.603530:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076572.243345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076572.627359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076572.627377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076572.627456:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076573.267397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076573.651326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076573.651333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076573.651408:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076574.291322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076574.675391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076574.675397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076574.675499:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076574.803374:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076574.803392:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076574.803400:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076574.803417:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3563 00000100:00080000:3.0:1769076574.803450:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076574.803462:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076574.803511:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617280640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076574.803547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076574.803551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076574.803571:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076574.803577:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3563 00000100:00080000:3.0:1769076574.803582:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076574.803589:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076574.803600:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076574.803621:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617280768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076574.803629:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617280640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076574.803661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076574.803669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076574.803673:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076574.803683:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3563 00000100:00080000:3.0:1769076574.803691:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076574.803700:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076574.803711:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076574.803737:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617280896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076574.803747:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076574.803763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076574.803769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076574.803772:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076574.803780:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3563 00000100:00080000:3.0:1769076574.803787:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076574.803793:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076574.803801:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076574.803842:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076574/real 1769076574] req@ffff9b0352a18380 x1855007617280640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076574.803881:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617281024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076574.803917:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617280640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076574.803918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076574.803934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076574.803940:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076574.803956:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3563 00000100:00080000:1.0:1769076574.803958:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076574.803962:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076574.803971:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769076574.803978:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076574.803987:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617280640:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769076574.803992:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076574.804016:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617281152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076574.804026:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076574.804076:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076574.804104:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076574/real 1769076574] req@ffff9b0352a1b800 x1855007617280768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076574.804120:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617280768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076574.804132:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076574.804136:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076574.804141:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076574.804158:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076574.804215:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076574.804281:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076574/real 1769076574] req@ffff9b0352a19500 x1855007617280896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076574.804308:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617280896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076574.804318:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076574.804323:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076574.804327:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617280896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076574.804335:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076574.804363:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076574.804376:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076574/real 1769076574] req@ffff9b0352a18000 x1855007617281024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076574.804390:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617281024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076574.804399:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076574.804403:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076574.804406:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076574.804426:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076574.804443:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076574.804453:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076574/real 1769076574] req@ffff9b0352a19c00 x1855007617281152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076574.804467:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617281152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076574.804476:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076574.804479:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076574.804482:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076575.315402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076575.699322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076575.699369:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076575.699369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076576.339358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076576.723400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076576.723401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076576.723503:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076577.363393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076577.747354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076577.747378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076577.747471:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076578.387349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076578.771372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076578.771377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076578.771502:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076579.411479:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076579.795360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076579.795366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076579.795440:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076579.923368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076579.923377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076579.923382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076579.923389:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3569 00000100:00080000:2.0:1769076579.923397:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076579.923402:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076579.923446:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617281280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076579.923470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076579.923486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076579.923487:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076579.923491:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3569 00000100:00080000:2.0:1769076579.923493:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076579.923496:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076579.923503:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076579.923513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617281408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076579.923520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076579.923523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076579.923524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076579.923526:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3569 00000100:00080000:2.0:1769076579.923528:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076579.923531:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076579.923534:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076579.923542:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617281536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076579.923550:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076579.923564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076579.923566:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076579.923569:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3569 00000100:00080000:2.0:1769076579.923571:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076579.923574:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076579.923578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076579.923584:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617281664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076579.923592:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076579.923595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076579.923596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076579.923603:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3569 00000100:00100000:1.0:1769076579.923605:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617281280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076579.923607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076579.923627:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076579.923630:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076579.923646:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617281792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076579.923809:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076579.923842:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076579.923871:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076579.923890:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076580.435338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076580.819355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076580.819355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076580.819382:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076581.459305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076581.843324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076581.843368:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076581.843396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076582.483315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076582.867392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076582.867411:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076582.867518:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769076582.995565:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769076582.995632:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076582.995670:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076582.995712:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076582.995752:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076582.995783:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076582.995792:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076582.995822:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076579/real 1769076582] req@ffff9b03482c4e00 x1855007617281664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076582.995859:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617281664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076582.995877:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076582.995884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076582.995890:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076582.995918:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076579/real 1769076582] req@ffff9b03482c7b80 x1855007617281792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076582.995939:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617281792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076582.995953:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076582.995959:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076582.995964:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076582.995974:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076579/real 1769076582] req@ffff9b03482c5880 x1855007617281280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076582.995995:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617281280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076582.996024:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076582.996029:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076582.996034:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617281280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076582.996043:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076579/real 1769076582] req@ffff9b03482c6680 x1855007617281408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076582.996059:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617281408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076582.996070:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076582.996075:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076582.996080:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076582.996089:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076579/real 1769076582] req@ffff9b03482c5c00 x1855007617281536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076582.996105:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617281536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076582.996116:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076582.996121:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076582.996127:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617281536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076583.507366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076583.891379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076583.891460:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076583.891484:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076584.531532:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076584.915417:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076584.915499:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076584.915574:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:2.0:1769076585.043466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076585.043478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076585.043483:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076585.043494:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3574 00000100:00080000:2.0:1769076585.043506:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076585.043513:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076585.043561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617281920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076585.043593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076585.043597:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076585.043599:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076585.043603:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3574 00000100:00080000:2.0:1769076585.043607:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076585.043610:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076585.043619:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076585.043632:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617282048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076585.043644:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076585.043648:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076585.043650:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076585.043655:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3574 00000100:00080000:2.0:1769076585.043658:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076585.043666:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617281920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076585.043676:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076585.043720:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076585.043742:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617282176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076585.043760:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076585.043765:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076585.043769:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076585.043774:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3574 00000100:00080000:2.0:1769076585.043780:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076585.043788:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076585.043796:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076585.043805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076585.043820:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617282304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076585.043837:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076585.043843:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076585.043846:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076585.043853:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3574 00000100:00080000:2.0:1769076585.043860:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076585.043871:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076585.043873:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076585] req@ffff9b03482c5f80 x1855007617281920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076585.043881:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076585.043912:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617282432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076585.043924:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617281920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076585.043951:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076585.043959:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076585.043968:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617281920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076585.043983:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076585.044023:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076585.044063:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076585] req@ffff9b03482c6a00 x1855007617282048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076585.044086:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617282048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076585.044112:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076585.044117:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076585.044123:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076585.044162:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076585.044201:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076585.044220:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076585] req@ffff9b03482c4000 x1855007617282176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076585.044300:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617282176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076585.044323:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076585.044328:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076585.044334:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076585.044344:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076585.044378:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076585.044397:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076585] req@ffff9b03482c7100 x1855007617282304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076585.044417:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617282304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076585.044429:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076585.044434:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076585.044439:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076585.044448:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076585.044479:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076585.044495:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076585/real 1769076585] req@ffff9b03482c4700 x1855007617282432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076585.044512:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617282432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076585.044537:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076585.044542:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076585.044548:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076585.555312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076585.939365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076585.939365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076585.939526:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076586.579391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076586.963342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076586.963389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076586.963411:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076587.603353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076587.987369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076587.987370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076587.987425:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076588.627331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076589.011395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076589.011434:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076589.011579:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076589.651416:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076590.035322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076590.035364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076590.035447:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076590.163500:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076590.163517:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076590.163524:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076590.163539:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3579 00000100:00080000:3.0:1769076590.163557:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076590.163568:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076590.163678:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617282560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076590.163724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076590.163731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076590.163733:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076590.163739:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3579 00000100:00080000:3.0:1769076590.163744:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076590.163750:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076590.163764:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076590.163782:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617282688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076590.163799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076590.163806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076590.163813:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076590.163818:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617282560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076590.163824:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3579 00000100:00080000:3.0:1769076590.163839:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076590.163846:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076590.163852:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076590.163868:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617282816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076590.163880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076590.163885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076590.163887:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076590.163892:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3579 00000100:00080000:3.0:1769076590.163899:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076590.163904:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076590.163912:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076590.163924:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617282944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076590.163938:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076590.163956:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076590.163962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076590.163964:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076590.163973:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3579 00000100:00080000:3.0:1769076590.163979:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076590.163985:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076590.163992:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076590.164013:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617283072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076590.164031:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076590/real 1769076590] req@ffff9b0352a19c00 x1855007617282560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076590.164055:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617282560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076590.164068:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076590.164074:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076590.164078:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617282560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076590.164105:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076590.164132:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076590.164143:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076590/real 1769076590] req@ffff9b0352a18000 x1855007617282688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076590.164155:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617282688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076590.164163:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076590.164166:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076590.164171:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076590.164177:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076590.164192:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076590.164200:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076590/real 1769076590] req@ffff9b0352a19500 x1855007617282816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076590.164210:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617282816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076590.164217:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076590.164219:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076590.164271:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076590.164278:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076590.164297:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076590.164308:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076590/real 1769076590] req@ffff9b0352a1b800 x1855007617282944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076590.164340:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617282944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076590.164350:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076590.164354:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076590.164358:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617282944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076590.164367:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076590.164415:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076590.164439:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076590/real 1769076590] req@ffff9b0352a18380 x1855007617283072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076590.164464:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617283072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076590.164479:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076590.164486:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076590.164493:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076590.675393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769076591.059354:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076591.059361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076591.059363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076591.699357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076592.083357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076592.083392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076592.083416:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076592.723343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076593.107346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076593.107375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076593.107390:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076593.747328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076594.131425:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076594.131475:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076594.131508:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076594.771379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076595.155312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076595.155375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076595.155421:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076595.283381:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076595.283393:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076595.283397:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076595.283407:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3584 00000100:00080000:1.0:1769076595.283415:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076595.283421:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076595.283452:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617283200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076595.283478:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076595.283481:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076595.283483:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076595.283486:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3584 00000100:00080000:1.0:1769076595.283488:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076595.283491:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076595.283498:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076595.283507:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617283328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076595.283535:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076595.283538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076595.283540:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076595.283543:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3584 00000100:00080000:1.0:1769076595.283546:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076595.283549:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076595.283553:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076595.283565:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617283456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076595.283575:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617283200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076595.283578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076595.283592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076595.283595:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076595.283600:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3584 00000100:00080000:1.0:1769076595.283605:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076595.283608:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076595.283616:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076595.283634:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617283584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076595.283645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076595.283648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076595.283650:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076595.283652:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3584 00000100:00080000:1.0:1769076595.283658:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076595.283660:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076595.283664:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076595.283674:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617283712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076595.283697:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076595.283780:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076595/real 1769076595] req@ffff9b0343deed80 x1855007617283200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076595.283823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617283200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076595.283836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076595.283842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076595.283846:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617283200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076595.283856:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076595.283896:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076595.283909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076595/real 1769076595] req@ffff9b0343deca80 x1855007617283328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076595.283923:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617283328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076595.283931:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076595.283935:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076595.283938:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076595.283967:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076595.283985:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076595.283994:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076595/real 1769076595] req@ffff9b0343dece00 x1855007617283456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076595.284005:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617283456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076595.284013:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076595.284016:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076595.284019:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076595.284024:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076595.284039:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076595.284047:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076595/real 1769076595] req@ffff9b0343defb80 x1855007617283584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076595.284056:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617283584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076595.284070:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076595.284073:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076595.284076:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076595.284081:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076595.284094:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076595.284101:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076595/real 1769076595] req@ffff9b0343dee680 x1855007617283712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076595.284110:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617283712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076595.284117:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076595.284119:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076595.284122:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076595.795346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076596.179350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076596.179387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076596.179392:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076596.819421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076597.203393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076597.203394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076597.203423:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076597.843377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076598.227355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076598.227386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076598.227429:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076598.867372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076599.251370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076599.251403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076599.251565:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076599.891368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076600.275391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076600.275409:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076600.275523:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076600.403497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076600.403506:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076600.403511:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076600.403522:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3589 00000100:00080000:1.0:1769076600.403533:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076600.403541:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076600.403604:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617283840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076600.403660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076600.403665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076600.403667:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076600.403673:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3589 00000100:00080000:1.0:1769076600.403678:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076600.403684:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076600.403696:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076600.403738:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617283968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076600.403752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076600.403761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1769076600.403765:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617283840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076600.403781:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076600.403798:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3589 00000100:00080000:1.0:1769076600.403811:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076600.403816:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076600.403820:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076600.403833:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617284096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076600.403842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076600.403844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076600.403845:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076600.403871:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3589 00000100:00080000:1.0:1769076600.403880:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076600.403885:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076600.403891:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076600.403903:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617284224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076600.403920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076600.403924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076600.403927:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076600.403932:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3589 00000100:00080000:1.0:1769076600.403936:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769076600.403946:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076600.403953:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076600.403961:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076600.403970:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617284352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076600.404034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076600/real 1769076600] req@ffff9b0343def480 x1855007617283840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076600.404068:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617283840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076600.404085:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076600.404094:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076600.404103:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617283840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076600.404149:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076600.404183:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076600.404216:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076600/real 1769076600] req@ffff9b0343ded500 x1855007617283968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076600.404277:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617283968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076600.404292:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076600.404300:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076600.404305:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617283968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076600.404314:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076600.404341:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076600.404357:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076600/real 1769076600] req@ffff9b0343dee680 x1855007617284096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076600.404379:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617284096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076600.404396:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076600.404403:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076600.404410:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076600.404423:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076600.404501:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076600.404532:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076600/real 1769076600] req@ffff9b0343defb80 x1855007617284224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076600.404563:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617284224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076600.404584:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076600.404594:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076600.404604:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076600.404622:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076600.404670:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076600.404705:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076600/real 1769076600] req@ffff9b0343dece00 x1855007617284352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076600.404744:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617284352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076600.404763:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076600.404771:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076600.404779:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076600.915460:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076601.299355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076601.299479:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076601.299495:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076601.939378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076602.323376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076602.323376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076602.323570:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076602.963330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076603.347333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076603.347333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076603.347434:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076603.987407:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076604.371452:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076604.371501:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076604.371528:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076605.011335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076605.395385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076605.395396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076605.395532:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076605.523331:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076605.523402:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076605.523411:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076605.523427:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3594 00000100:00080000:3.0:1769076605.523441:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076605.523453:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076605.523509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617284480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076605.523552:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076605.523576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076605.523579:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076605.523597:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3594 00000100:00080000:3.0:1769076605.523604:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076605.523611:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076605.523622:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617284480:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769076605.523624:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076605.523644:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617284608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076605.523660:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076605.523665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076605.523668:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076605.523673:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3594 00000100:00080000:3.0:1769076605.523677:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076605.523682:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076605.523689:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076605.523701:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076605.523717:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617284736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076605.523744:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076605.523748:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076605.523751:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076605.523757:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3594 00000100:00080000:3.0:1769076605.523763:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076605.523767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076605/real 1769076605] req@ffff9b0352a1b100 x1855007617284480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076605.523768:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076605.523776:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076605.523791:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617284864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076605.523793:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617284480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076605.523808:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076605.523810:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076605.523829:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076605.523832:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076605.523833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769076605.523837:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3594 00000100:00100000:0.0:1769076605.523838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617284480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076605.523842:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076605.523848:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076605.523853:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076605.523865:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617284992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076605.523868:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076605.523900:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076605.523919:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076605/real 1769076605] req@ffff9b0352a1ad80 x1855007617284608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076605.523939:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617284608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076605.523953:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076605.523958:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076605.523964:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076605.523973:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076605.524005:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076605.524023:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076605/real 1769076605] req@ffff9b0352a1b480 x1855007617284736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076605.524042:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617284736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076605.524050:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076605.524053:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076605.524056:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076605.524063:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076605.524084:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076605.524094:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076605/real 1769076605] req@ffff9b0352a18a80 x1855007617284864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076605.524106:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617284864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076605.524113:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076605.524116:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076605.524119:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076605.524141:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076605.524168:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076605.524185:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076605/real 1769076605] req@ffff9b0352a1a680 x1855007617284992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076605.524206:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617284992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076605.524219:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076605.524223:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076605.524226:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617284992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076606.035455:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076606.419450:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076606.419543:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076606.419553:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076607.059393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076607.443385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076607.443423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076607.443553:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076608.083439:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076608.467349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076608.467367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076608.467446:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076609.107386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076609.491356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076609.491359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076609.491452:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076610.131360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076610.515370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076610.515405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076610.515466:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076610.643450:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076610.643464:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076610.643469:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076610.643481:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3599 00000100:00080000:1.0:1769076610.643494:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076610.643503:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076610.643552:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617285120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076610.643624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076610.643629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076610.643632:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076610.643638:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3599 00000100:00080000:1.0:1769076610.643643:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076610.643650:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076610.643662:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076610.643681:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617285120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076610.643683:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617285248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076610.643708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076610.643713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076610.643715:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076610.643721:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3599 00000100:00080000:1.0:1769076610.643725:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076610.643729:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076610.643735:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076610.643749:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617285376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076610.643760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076610.643764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076610.643766:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076610.643775:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3599 00000800:00000100:0.0:1769076610.643776:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076610.643785:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076610.643793:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076610.643801:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076610.643816:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617285504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076610.643828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076610.643833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076610.643839:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076610/real 1769076610] req@ffff9b0343deca80 x1855007617285120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076610.643840:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076610.643855:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3599 00000100:00080000:1.0:1769076610.643865:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076610.643874:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076610.643883:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076610.643890:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617285120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076610.643899:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617285632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076610.643923:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076610.643937:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076610.643947:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617285120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076610.643984:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617285248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076610.644027:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076610.644050:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076610/real 1769076610] req@ffff9b0343deed80 x1855007617285248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076610.644067:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617285248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076610.644085:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076610.644089:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076610.644094:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617285248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076610.644101:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617285376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076610.644125:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076610.644136:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076610/real 1769076610] req@ffff9b0343deea00 x1855007617285376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076610.644150:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617285376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076610.644159:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076610.644162:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076610.644165:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617285376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076610.644172:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617285504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076610.644189:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076610.644199:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076610/real 1769076610] req@ffff9b0343def800 x1855007617285504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076610.644212:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617285504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076610.644221:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076610.644224:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076610.644228:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617285504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076610.644290:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617285632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076610.644338:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076610.644353:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076610/real 1769076610] req@ffff9b0343ded880 x1855007617285632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076610.644368:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617285632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076610.644379:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076610.644382:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076610.644386:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617285632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076611.155403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076611.539357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076611.539384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076611.539464:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076612.179389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076612.563370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076612.563399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076612.563501:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076613.203341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076613.587400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076613.587403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076613.587479:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076614.227363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076614.611420:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076614.611436:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076614.611494:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076615.251460:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076615.635369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076615.635378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076615.635449:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076615.763455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076615.763468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076615.763474:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076615.763485:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3604 00000100:00080000:2.0:1769076615.763510:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076615.763519:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076615.763567:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617285760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076615.763610:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076615.763615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076615.763617:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076615.763623:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3604 00000100:00080000:2.0:1769076615.763627:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076615.763631:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076615.763642:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076615.763656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617285888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076615.763668:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076615.763672:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076615.763675:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076615.763679:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3604 00000100:00080000:2.0:1769076615.763682:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076615.763686:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076615.763691:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076615.763704:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617286016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076615.763715:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076615.763720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076615.763723:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076615.763727:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3604 00000100:00080000:2.0:1769076615.763731:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076615.763736:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076615.763741:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076615.763752:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617286144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076615.763766:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617285760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076615.763770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076615.763782:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076615.763789:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076615.763795:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3604 00000100:00080000:2.0:1769076615.763799:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076615.763803:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076615.763809:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076615.763826:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617286272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076615.763888:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076615.763947:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076615/real 1769076615] req@ffff9b03482c6d80 x1855007617285760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076615.763971:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617285760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076615.763985:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076615.763991:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076615.763996:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617285760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076615.764006:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617285888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076615.764054:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076615.764080:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076615/real 1769076615] req@ffff9b03482c6300 x1855007617285888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076615.764106:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617285888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076615.764135:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076615.764141:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076615.764149:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617285888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076615.764161:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076615.764278:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076615.764295:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076615/real 1769076615] req@ffff9b03482c4a80 x1855007617286016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076615.764312:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617286016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076615.764322:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076615.764326:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076615.764330:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076615.764362:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076615.764386:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076615.764398:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076615/real 1769076615] req@ffff9b03482c5180 x1855007617286144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076615.764412:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617286144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076615.764423:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076615.764427:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076615.764431:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076615.764437:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076615.764454:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076615.764464:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076615/real 1769076615] req@ffff9b03482c4380 x1855007617286272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076615.764477:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617286272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076615.764487:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076615.764490:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076615.764494:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076616.275421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076616.659313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076616.659313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076616.659369:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076617.299337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076617.683324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076617.683366:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076617.683423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076618.323362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076618.707329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076618.707356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076618.707514:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076619.347368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076619.731352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076619.731365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076619.731417:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076620.371356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076620.755320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076620.755331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076620.755386:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076620.883346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076620.883356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076620.883361:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076620.883370:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3610 00000100:00080000:2.0:1769076620.883378:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076620.883384:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076620.883419:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617286400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076620.883443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076620.883446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076620.883447:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076620.883451:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3610 00000100:00080000:2.0:1769076620.883453:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076620.883456:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076620.883465:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076620.883474:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617286400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076620.883477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617286528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076620.883488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076620.883491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076620.883493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076620.883498:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3610 00000100:00080000:2.0:1769076620.883502:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076620.883507:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076620.883512:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076620.883536:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617286656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076620.883550:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076620.883561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076620.883565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076620.883568:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076620.883574:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3610 00000100:00080000:2.0:1769076620.883579:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076620.883583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076620.883589:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076620/real 1769076620] req@ffff9b03482c5500 x1855007617286400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076620.883590:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076620.883609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617286784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076620.883611:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617286400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076620.883638:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076620.883640:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076620.883644:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:2.0:1769076620.883649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076620.883650:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617286400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076620.883653:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076620.883660:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3610 00000100:00080000:2.0:1769076620.883665:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076620.883670:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076620.883673:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286528:192.168.204.160@tcp:38:kworker.0 00000008:00100000:2.0:1769076620.883679:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076620.883694:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617286912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076620.883696:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076620.883710:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076620/real 1769076620] req@ffff9b03482c4380 x1855007617286528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076620.883722:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617286528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076620.883729:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076620.883731:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076620.883733:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076620.883738:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076620.883748:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076620.883754:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076620/real 1769076620] req@ffff9b03482c5180 x1855007617286656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076620.883761:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617286656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076620.883766:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076620.883768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076620.883770:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076620.883773:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076620.883788:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076620.883799:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076620/real 1769076620] req@ffff9b03482c4a80 x1855007617286784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076620.883812:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617286784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076620.883821:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076620.883824:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076620.883828:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076620.883843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076620.883876:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076620.883884:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076620/real 1769076620] req@ffff9b03482c6300 x1855007617286912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076620.883892:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617286912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076620.883898:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076620.883900:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076620.883902:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617286912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076621.395346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076621.779322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076621.779325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076621.779421:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076622.419332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076622.803367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076622.803432:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076622.803570:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076623.443349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076623.827390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076623.827390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076623.827426:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076624.467354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076624.851384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076624.851390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076624.851590:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076625.491354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076625.875368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076625.875394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076625.875470:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076626.003461:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076626.003473:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076626.003479:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076626.003491:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3615 00000100:00080000:3.0:1769076626.003503:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076626.003512:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076626.003560:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617287040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076626.003600:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076626.003605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076626.003608:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076626.003613:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3615 00000100:00080000:3.0:1769076626.003617:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076626.003622:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076626.003634:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076626.003651:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617287168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076626.003666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076626.003673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076626.003677:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076626.003680:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617287040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076626.003685:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3615 00000100:00080000:3.0:1769076626.003696:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076626.003706:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076626.003712:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076626.003728:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617287296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076626.003739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076626.003743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076626.003747:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076626.003753:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3615 00000100:00080000:3.0:1769076626.003760:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076626.003764:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076626.003770:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076626.003786:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617287424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076626.003797:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076626.003813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076626.003818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076626.003821:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076626.003842:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3615 00000100:00080000:3.0:1769076626.003849:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076626.003855:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076626.003862:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076626.003863:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9b0352a18700 x1855007617287040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076626.003885:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617287552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076626.003901:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617287040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.003928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.003935:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076626.003941:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617287040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076626.003971:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076626.004050:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076626.004092:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9b0352a18e00 x1855007617287168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076626.004117:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617287168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.004134:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.004140:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076626.004147:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076626.004159:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076626.004201:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076626.004224:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9b0352a1a300 x1855007617287296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076626.004280:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617287296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.004292:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.004296:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076626.004301:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076626.004310:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076626.004340:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076626.004354:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9b0352a19f80 x1855007617287424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076626.004368:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617287424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.004379:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.004382:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076626.004386:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076626.004393:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076626.004410:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076626.004420:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076626/real 1769076626] req@ffff9b0352a1aa00 x1855007617287552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076626.004434:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617287552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076626.004443:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076626.004446:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076626.004450:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076626.515347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076626.899401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076626.899420:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076626.900224:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076627.539366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076627.923440:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076627.923679:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076627.923995:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076628.563323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076628.947335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076628.947371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076628.947822:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076629.587355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076629.971602:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076629.971706:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076629.971776:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076630.611306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076630.995609:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076630.995638:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076630.995642:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769076631.123363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076631.123375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076631.123379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076631.123387:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3620 00000100:00080000:2.0:1769076631.123396:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076631.123403:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076631.123439:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617287680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076631.123465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076631.123469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076631.123471:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076631.123476:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3620 00000100:00080000:2.0:1769076631.123481:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076631.123488:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076631.123498:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076631.123505:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617287680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076631.123520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617287808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076631.123537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076631.123541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076631.123544:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076631.123550:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3620 00000100:00080000:2.0:1769076631.123555:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076631.123559:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076631.123564:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076631.123576:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617287936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076631.123581:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076631.123592:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076631.123595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076631.123597:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076631.123602:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3620 00000100:00080000:2.0:1769076631.123606:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076631.123609:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076631.123613:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076631.123624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617288064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076631.123628:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076631/real 1769076631] req@ffff9b03482c6d80 x1855007617287680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076631.123650:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076631.123654:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076631.123656:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769076631.123663:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617287680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076631.123664:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3620 00000100:00080000:2.0:1769076631.123670:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076631.123678:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076631.123684:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1769076631.123685:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076631.123694:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076631.123703:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617287680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076631.123703:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617288192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076631.123735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076631.123755:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076631.123765:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076631/real 1769076631] req@ffff9b03482c4700 x1855007617287808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076631.123775:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617287808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076631.123782:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076631.123784:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076631.123787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076631.123791:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076631.123802:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076631.123808:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076631/real 1769076631] req@ffff9b03482c7100 x1855007617287936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076631.123817:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617287936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076631.123828:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076631.123831:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076631.123833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617287936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076631.123837:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076631.123848:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076631.123855:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076631/real 1769076631] req@ffff9b03482c4000 x1855007617288064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076631.123863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617288064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076631.123868:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076631.123870:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076631.123873:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076631.123877:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076631.123887:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076631.123893:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076631/real 1769076631] req@ffff9b03482c6a00 x1855007617288192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076631.123901:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617288192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076631.123907:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076631.123934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076631.123938:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076631.635351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769076632.019529:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076632.019863:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076632.019873:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076632.659330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076633.043341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076633.043342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076633.043506:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076633.683349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076634.067317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076634.067392:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076634.067411:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076634.707393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076635.091374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076635.091424:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076635.091456:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076635.731349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076636.115392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076636.115392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076636.115414:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076636.243420:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076636.243437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076636.243444:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076636.243476:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3625 00000100:00080000:3.0:1769076636.243491:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076636.243501:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076636.243547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617288320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076636.243613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076636.243622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076636.243626:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076636.243660:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3625 00000100:00080000:3.0:1769076636.243668:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076636.243676:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076636.243692:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076636.243747:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617288320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076636.243754:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617288448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076636.243934:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076636.243990:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076636/real 1769076636] req@ffff9b0352a1aa00 x1855007617288320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076636.244016:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617288320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076636.244029:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076636.244035:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076636.244041:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617288320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076636.244127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076636.244135:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076636.244140:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076636.244168:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3625 00000100:00080000:3.0:1769076636.244178:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076636.244187:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076636.244198:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076636.244201:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076636.244233:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617288576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076636.244319:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076636.244372:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076636/real 1769076636] req@ffff9b0352a19f80 x1855007617288448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076636.244397:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617288448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076636.244415:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076636.244421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076636.244427:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769076636.244430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076636.244438:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076636.244443:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288448:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076636.244449:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3625 00000100:00080000:3.0:1769076636.244462:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076636.244471:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076636.244480:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076636.244487:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076636.244504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617288704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076636.244520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076636.244525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076636.244528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076636.244534:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3625 00000100:00080000:3.0:1769076636.244540:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076636.244543:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076636.244550:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076636.244559:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076636.244580:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076636/real 1769076636] req@ffff9b0352a1a300 x1855007617288576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076636.244589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617288832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076636.244631:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617288576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076636.244655:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076636.244660:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076636.244664:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076636.244683:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076636.244707:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076636.244718:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076636/real 1769076636] req@ffff9b0352a18e00 x1855007617288704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076636.244742:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617288704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076636.244750:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076636.244754:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076636.244757:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076636.244763:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076636.244778:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076636.244788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076636/real 1769076636] req@ffff9b0352a18700 x1855007617288832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076636.244808:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617288832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076636.244816:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076636.244818:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076636.244821:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617288832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076636.755418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076637.139336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076637.139336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076637.139432:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076637.779315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076638.163351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076638.163386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076638.163430:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076638.803359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076639.187356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076639.187382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076639.187452:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076639.827358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076640.211366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076640.211375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076640.211478:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076640.851418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076641.235338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076641.235364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076641.235406:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076641.363384:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076641.363423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076641.363430:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076641.363443:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3630 00000100:00080000:3.0:1769076641.363456:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076641.363464:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076641.363513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617288960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076641.363568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076641.363573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076641.363578:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076641.363585:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3630 00000100:00080000:3.0:1769076641.363590:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076641.363606:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076641.363619:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076641.363624:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617288960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076641.363640:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617289088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076641.363658:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076641.363663:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076641.363666:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076641.363672:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3630 00000100:00080000:3.0:1769076641.363677:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076641.363682:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076641.363688:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076641.363704:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617289216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076641.363717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076641.363728:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076641.363728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076641.363736:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076641.363769:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3630 00000100:00080000:3.0:1769076641.363776:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076641.363782:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076641.363791:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076641.363810:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617289344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076641.363823:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076641.363830:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769076641.363836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076641/real 1769076641] req@ffff9b0352a1a680 x1855007617288960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076641.363837:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076641.363856:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3630 00000100:00080000:3.0:1769076641.363861:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076641.363865:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076641.363875:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076641.363899:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617289472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076641.363913:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617288960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076641.363931:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076641.363940:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076641.363947:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617288960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076641.363981:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076641.364016:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076641.364033:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076641/real 1769076641] req@ffff9b0352a18a80 x1855007617289088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076641.364049:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617289088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076641.364060:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076641.364064:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076641.364068:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076641.364076:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076641.364095:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076641.364105:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076641/real 1769076641] req@ffff9b0352a1b480 x1855007617289216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076641.364141:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617289216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076641.364152:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076641.364156:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076641.364160:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076641.364167:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076641.364188:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076641.364199:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076641/real 1769076641] req@ffff9b0352a1ad80 x1855007617289344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076641.364214:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617289344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076641.364224:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076641.364228:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076641.364262:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076641.364271:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076641.364294:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076641.364306:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076641/real 1769076641] req@ffff9b0352a1b100 x1855007617289472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076641.364321:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617289472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076641.364331:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076641.364334:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076641.364338:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076641.875417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076642.259348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076642.259435:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076642.259470:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076642.899352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076643.283326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076643.283334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076643.283445:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076643.923405:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076644.307349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076644.307382:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076644.307402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076644.947396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076645.331327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076645.331342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076645.331392:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076645.971345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076646.355308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076646.355310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076646.355340:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076646.483350:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076646.483360:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076646.483364:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076646.483372:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3635 00000100:00080000:1.0:1769076646.483380:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076646.483385:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076646.483424:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617289600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076646.483445:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076646.483448:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076646.483450:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076646.483453:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3635 00000100:00080000:1.0:1769076646.483455:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076646.483458:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076646.483467:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076646.483480:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617289728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076646.483486:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617289600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076646.483492:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076646.483496:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076646.483497:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076646.483500:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3635 00000100:00080000:1.0:1769076646.483502:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076646.483505:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076646.483509:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076646.483516:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617289856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076646.483523:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076646.483526:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076646.483527:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076646.483529:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3635 00000100:00080000:1.0:1769076646.483532:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076646.483534:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076646.483537:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076646.483544:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617289984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076646.483550:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076646.483552:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076646.483554:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076646.483556:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3635 00000100:00080000:1.0:1769076646.483559:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076646.483561:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076646.483565:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076646.483573:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617290112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076646.483599:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076646.483637:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076646.483678:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076646.483702:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617290112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076646.995297:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076647.379293:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076647.379328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076647.379398:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076648.019304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076648.403381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076648.403414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076648.403416:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076649.043283:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076649.427365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076649.427411:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076649.427508:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000400:00000100:0.0:1769076649.555588:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769076649.555823:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769076649.555846:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076649.555892:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076649.555902:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076649.555912:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076649.555923:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769076649.555960:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076646/real 1769076649] req@ffff9b0343dedf80 x1855007617289728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076649.555996:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617289728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076649.556009:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076649.556014:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076649.556018:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076649.556025:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076646/real 1769076649] req@ffff9b0343def100 x1855007617289856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076649.556035:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617289856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076649.556041:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076649.556043:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076649.556045:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076649.556050:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076646/real 1769076649] req@ffff9b0343dee300 x1855007617289984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076649.556058:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617289984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076649.556065:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076649.556068:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076649.556070:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617289984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076649.556075:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076646/real 1769076649] req@ffff9b0343dec380 x1855007617290112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076649.556083:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617290112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076649.556088:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076649.556090:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076649.556092:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617290112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076649.556097:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076646/real 1769076649] req@ffff9b0343ded180 x1855007617289600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076649.556110:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617289600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076649.556117:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076649.556120:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076649.556124:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617289600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076650.067300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076650.451372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076650.451376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076650.451514:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076651.091328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076651.475307:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076651.475369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076651.475378:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076651.603371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076651.603384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076651.603390:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076651.603402:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3640 00000100:00080000:2.0:1769076651.603414:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076651.603421:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076651.603460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617290240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076651.603500:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076651.603505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076651.603507:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076651.603513:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3640 00000100:00080000:2.0:1769076651.603517:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076651.603521:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076651.603533:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076651.603556:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617290240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076651.603557:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617290368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076651.603597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076651.603603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076651.603605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076651.603609:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3640 00000100:00080000:2.0:1769076651.603615:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076651.603620:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076651.603627:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076651.603647:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617290496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076651.603662:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076651.603669:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076651.603673:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076651.603675:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076651.603681:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3640 00000100:00080000:2.0:1769076651.603686:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076651.603689:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076651.603695:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076651.603709:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617290624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076651.603712:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076651/real 1769076651] req@ffff9b03482c5f80 x1855007617290240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076651.603728:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076651.603733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076651.603736:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076651.603741:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3640 00000100:00080000:2.0:1769076651.603745:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076651.603752:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076651.603755:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617290240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769076651.603757:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076651.603773:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617290752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076651.603789:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076651.603797:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076651.603805:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617290240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076651.603833:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617290368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076651.603860:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076651.603874:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076651/real 1769076651] req@ffff9b03482c5c00 x1855007617290368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076651.603888:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617290368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076651.603896:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076651.603899:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076651.603902:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617290368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076651.603908:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617290496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076651.603928:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076651.603940:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076651/real 1769076651] req@ffff9b03482c6680 x1855007617290496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076651.603952:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617290496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076651.603962:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076651.603966:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076651.603970:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617290496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076651.603976:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617290624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076651.603997:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076651.604009:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076651/real 1769076651] req@ffff9b03482c5880 x1855007617290624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076651.604022:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617290624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076651.604031:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076651.604034:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076651.604038:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617290624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076651.604044:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617290752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076651.604063:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076651.604074:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076651/real 1769076651] req@ffff9b03482c7b80 x1855007617290752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076651.604086:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617290752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076651.604094:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076651.604097:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076651.604100:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617290752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076652.115318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076652.499384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076652.499396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076652.499452:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076653.139283:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076653.523307:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076653.523308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076653.523401:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076654.163287:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076654.547348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076654.547349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076654.547406:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076655.187330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076655.571333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076655.571387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076655.571437:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076656.211302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076656.595389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076656.595390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076656.595469:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076656.723365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076656.723374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076656.723378:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076656.723385:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3645 00000100:00080000:3.0:1769076656.723393:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076656.723399:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076656.723431:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617290880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076656.723453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076656.723456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076656.723457:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076656.723460:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3645 00000100:00080000:3.0:1769076656.723463:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076656.723465:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076656.723473:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076656.723497:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617291008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076656.723502:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617290880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076656.723518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076656.723522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076656.723525:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076656.723532:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3645 00000100:00080000:3.0:1769076656.723536:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076656.723541:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076656.723547:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076656.723560:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617291136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076656.723567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076656.723568:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076656.723573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076656.723575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076656.723579:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3645 00000100:00080000:3.0:1769076656.723583:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076656.723586:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076656.723591:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076656.723598:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617291264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076656.723606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076656.723609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769076656.723610:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076656/real 1769076656] req@ffff9b0352a18380 x1855007617290880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076656.723611:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076656.723618:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3645 00000100:00080000:3.0:1769076656.723621:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076656.723627:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076656.723633:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076656.723643:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617291392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076656.723648:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617290880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076656.723661:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076656.723665:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076656.723669:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617290880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076656.723688:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076656.723706:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076656.723715:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076656/real 1769076656] req@ffff9b0352a1b800 x1855007617291008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076656.723724:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617291008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076656.723729:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076656.723732:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076656.723734:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076656.723738:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076656.723749:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076656.723754:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076656/real 1769076656] req@ffff9b0352a19500 x1855007617291136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076656.723761:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617291136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076656.723767:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076656.723769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076656.723771:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076656.723776:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076656.723794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076656.723816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076656/real 1769076656] req@ffff9b0352a18000 x1855007617291264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076656.723829:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617291264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076656.723838:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076656.723841:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076656.723844:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076656.723848:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076656.723865:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076656.723876:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076656/real 1769076656] req@ffff9b0352a19c00 x1855007617291392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076656.723888:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617291392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076656.723897:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076656.723901:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076656.723904:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076657.235389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076657.619341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076657.619368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076657.619425:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076658.259311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076658.643326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076658.643326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076658.643383:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076659.283304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076659.667314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076659.667324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076659.667402:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076660.307335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076660.691309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076660.691319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076660.691964:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076661.331319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076661.715312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076661.715315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076661.715369:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076661.843380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076661.843390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076661.843411:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076661.843419:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3651 00000100:00080000:2.0:1769076661.843427:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076661.843433:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076661.843473:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617291520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076661.843494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076661.843497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076661.843499:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076661.843502:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3651 00000100:00080000:2.0:1769076661.843504:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076661.843507:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076661.843515:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076661.843526:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617291648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076661.843531:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617291520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076661.843543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076661.843546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076661.843548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076661.843551:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3651 00000100:00080000:2.0:1769076661.843554:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076661.843557:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076661.843560:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076661.843569:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617291776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076661.843575:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076661.843579:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076661.843581:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076661.843585:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3651 00000100:00080000:2.0:1769076661.843590:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076661.843594:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076661.843599:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769076661.843603:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076661.843616:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617291904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076661.843627:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076661.843631:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076661.843633:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076661.843637:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3651 00000100:00080000:2.0:1769076661.843642:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076661.843648:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076661/real 1769076661] req@ffff9b03482c4e00 x1855007617291520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076661.843649:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076661.843657:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076661.843671:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617292032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076661.843679:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617291520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076661.843694:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076661.843699:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076661.843704:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617291520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076661.843724:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076661.843743:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076661.843752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076661/real 1769076661] req@ffff9b03482c7b80 x1855007617291648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076661.843760:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617291648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076661.843766:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076661.843768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076661.843770:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076661.843774:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076661.843794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076661.843801:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076661/real 1769076661] req@ffff9b03482c5880 x1855007617291776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076661.843809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617291776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076661.843818:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076661.843821:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076661.843823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076661.843826:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076661.843837:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076661.843842:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076661/real 1769076661] req@ffff9b03482c6680 x1855007617291904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076661.843849:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617291904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076661.843854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076661.843856:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076661.843858:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617291904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076661.843861:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076661.843870:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076661.843876:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076661/real 1769076661] req@ffff9b03482c5c00 x1855007617292032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076661.843883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617292032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076661.843887:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076661.843889:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076661.843891:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076662.355338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076662.739340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076662.739371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076662.739422:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076663.379340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076663.763354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076663.763394:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076663.763458:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076664.403342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076664.787391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076664.787395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076664.787459:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076665.427321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076665.811376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076665.811385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076665.811453:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076666.451369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076666.835357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076666.835357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076666.835417:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076666.963375:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076666.963383:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076666.963387:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076666.963395:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3656 00000100:00080000:3.0:1769076666.963403:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076666.963416:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076666.963450:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617292160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076666.963478:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076666.963481:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076666.963482:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076666.963485:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3656 00000100:00080000:3.0:1769076666.963488:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076666.963490:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076666.963498:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076666.963507:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617292288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076666.963514:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076666.963519:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1769076666.963521:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617292160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076666.963521:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076666.963527:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3656 00000100:00080000:3.0:1769076666.963532:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076666.963535:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076666.963539:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076666.963547:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617292416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076666.963553:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076666.963556:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076666.963557:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076666.963559:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3656 00000100:00080000:3.0:1769076666.963562:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076666.963566:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076666.963569:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076666.963575:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617292544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076666.963582:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076666.963585:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076666.963586:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076666.963591:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3656 00000800:00000100:0.0:1769076666.963593:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076666.963595:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076666.963600:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076666.963602:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076666.963610:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617292672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076666.963642:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076666/real 1769076666] req@ffff9b0352a19c00 x1855007617292160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076666.963661:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617292160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076666.963672:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076666.963676:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076666.963680:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617292160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076666.963703:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076666.963724:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076666.963735:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076666/real 1769076666] req@ffff9b0352a18000 x1855007617292288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076666.963746:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617292288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076666.963755:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076666.963758:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076666.963762:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076666.963767:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076666.963784:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076666.963796:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076666/real 1769076666] req@ffff9b0352a19500 x1855007617292416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076666.963807:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617292416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076666.963816:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076666.963819:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076666.963823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076666.963828:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076666.963851:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076666.963862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076666/real 1769076666] req@ffff9b0352a1b800 x1855007617292544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076666.963886:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617292544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076666.963895:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076666.963899:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076666.963902:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076666.963908:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076666.963929:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076666.963942:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076666/real 1769076666] req@ffff9b0352a18380 x1855007617292672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076666.963955:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617292672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076666.963963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076666.963967:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076666.963970:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076667.475316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769076667.859380:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076667.859381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076667.859390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076668.499365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076668.883369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076668.883370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076668.883428:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076669.523329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076669.907332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076669.907357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076669.907407:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076670.547338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076670.931342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076670.931348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076670.931416:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076671.571448:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076671.955403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076671.955412:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076671.955531:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076672.083405:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076672.083420:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076672.083427:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076672.083442:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3661 00000100:00080000:2.0:1769076672.083460:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076672.083471:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076672.083541:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617292800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076672.083609:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076672.083629:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076672.083635:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076672.083647:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3661 00000100:00080000:2.0:1769076672.083673:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076672.083683:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076672.083702:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617292800:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769076672.083709:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076672.083749:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617292928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076672.083768:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076672.083774:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076672.083777:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076672.083782:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3661 00000100:00080000:2.0:1769076672.083788:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076672.083794:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076672.083801:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769076672.083811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076672.083826:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617293056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076672.083845:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076672.083852:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076672.083856:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076672.083865:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3661 00000100:00080000:2.0:1769076672.083872:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076672.083879:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076672/real 1769076672] req@ffff9b03482c5f80 x1855007617292800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076672.083887:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076672.083908:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076672.083929:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617293184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076672.083932:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617292800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076672.083983:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076672.083988:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076672.083990:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076672.083999:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076672.084012:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3661 00000100:00080000:0.0:1769076672.084021:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:2.0:1769076672.084022:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076672.084042:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617292800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076672.084043:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076672.084061:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076672.084092:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617293312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076672.084101:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076672.084167:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076672.084189:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076672/real 1769076672] req@ffff9b03482c6a00 x1855007617292928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076672.084213:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617292928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076672.084305:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076672.084317:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076672.084329:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617292928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076672.084344:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076672.084381:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076672.084400:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076672/real 1769076672] req@ffff9b03482c4000 x1855007617293056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076672.084419:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617293056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076672.084427:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076672.084430:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076672.084434:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076672.084441:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076672.084462:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076672.084472:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076672/real 1769076672] req@ffff9b03482c7100 x1855007617293184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076672.084483:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617293184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076672.084491:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076672.084494:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076672.084497:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076672.084514:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076672.084529:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076672.084537:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076672/real 1769076672] req@ffff9b03482c4700 x1855007617293312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076672.084548:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617293312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076672.084564:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076672.084567:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076672.084570:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076672.595383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076672.979353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076672.979428:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076672.979531:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076673.619316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076674.003447:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076674.003513:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076674.003518:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076674.643412:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076675.027333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076675.027383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076675.027510:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076675.667373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076676.051361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076676.051361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076676.051425:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076676.691346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076677.075433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076677.075433:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076677.075611:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076677.203366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076677.203382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076677.203388:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076677.203403:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3666 00000100:00080000:2.0:1769076677.203416:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076677.203426:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076677.203492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617293440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076677.203524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076677.203528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076677.203531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076677.203535:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3666 00000100:00080000:2.0:1769076677.203539:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076677.203546:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076677.203557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076677.203571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617293568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076677.203578:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617293440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076677.203589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076677.203594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076677.203596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076677.203600:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3666 00000100:00080000:2.0:1769076677.203603:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076677.203607:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076677.203612:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076677.203629:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617293696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076677.203643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076677.203649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076677.203651:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076677.203657:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3666 00000100:00080000:2.0:1769076677.203662:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076677.203669:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076677.203678:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769076677.203686:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076677.203698:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617293824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076677.203715:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076677.203720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076677.203722:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076677.203727:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3666 00000100:00080000:2.0:1769076677.203733:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076677.203737:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076677.203743:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076677.203744:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076677/real 1769076677] req@ffff9b03482c6d80 x1855007617293440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076677.203761:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617293952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076677.203803:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617293440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076677.203817:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076677.203823:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076677.203829:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617293440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076677.203880:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076677.203904:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076677.203944:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076677/real 1769076677] req@ffff9b03482c6300 x1855007617293568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076677.203957:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617293568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076677.203978:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076677.203984:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076677.203989:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076677.203998:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076677.204066:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076677.204082:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076677/real 1769076677] req@ffff9b03482c4a80 x1855007617293696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076677.204097:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617293696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076677.204105:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076677.204108:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076677.204111:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076677.204117:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076677.204170:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076677.204183:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076677/real 1769076677] req@ffff9b03482c5180 x1855007617293824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076677.204203:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617293824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076677.204210:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076677.204213:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076677.204217:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076677.204223:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076677.204288:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076677.204301:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076677/real 1769076677] req@ffff9b03482c4380 x1855007617293952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076677.204313:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617293952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076677.204322:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076677.204325:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076677.204329:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617293952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076677.715333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076678.099355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076678.099572:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076678.099587:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076678.739373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076679.123445:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076679.123545:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076679.123549:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076679.763395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076680.147386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076680.147471:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076680.147508:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076680.787345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076681.171377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076681.171377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076681.171492:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076681.811320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076682.195332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076682.195332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076682.195409:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076682.323350:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076682.323359:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076682.323363:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076682.323371:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3671 00000100:00080000:1.0:1769076682.323379:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076682.323384:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076682.323416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617294080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076682.323439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076682.323441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076682.323443:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076682.323446:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3671 00000100:00080000:1.0:1769076682.323449:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076682.323452:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076682.323460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076682.323473:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617294080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076682.323473:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617294208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076682.323509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076682.323514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076682.323515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076682.323520:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3671 00000100:00080000:1.0:1769076682.323539:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076682.323545:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769076682.323551:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769076682.323553:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076682.323570:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617294336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076682.323579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076682.323583:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076682.323584:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076682.323589:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3671 00000100:00080000:1.0:1769076682.323593:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076682.323598:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076682/real 1769076682] req@ffff9b0343dec700 x1855007617294080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076682.323600:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076682.323610:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076682.323621:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617294080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076682.323626:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617294464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076682.323634:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076682.323643:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769076682.323643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076682.323650:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617294080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076682.323651:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076682.323656:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076682.323662:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3671 00000100:00080000:1.0:1769076682.323666:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076682.323670:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076682.323675:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294208:192.168.204.160@tcp:38:kworker.0 00000008:00100000:1.0:1769076682.323676:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076682.323695:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617294592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076682.323703:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076682.323720:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076682/real 1769076682] req@ffff9b0343dec000 x1855007617294208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076682.323730:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617294208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076682.323736:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076682.323739:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076682.323741:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076682.323745:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076682.323756:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076682.323762:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076682/real 1769076682] req@ffff9b0343ded180 x1855007617294336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076682.323769:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617294336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076682.323774:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076682.323776:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076682.323779:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076682.323782:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076682.323791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076682.323797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076682/real 1769076682] req@ffff9b0343dec380 x1855007617294464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076682.323804:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617294464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076682.323809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076682.323811:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076682.323813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076682.323822:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076682.323831:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076682.323837:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076682/real 1769076682] req@ffff9b0343dee300 x1855007617294592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076682.323844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617294592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076682.323849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076682.323851:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076682.323852:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076682.835383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076683.219312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076683.219317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076683.219422:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076683.859301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076684.243319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076684.243358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076684.243476:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076684.883415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076685.267384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076685.267404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076685.267450:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076685.907401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076686.291366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076686.291428:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076686.291559:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076686.931330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076687.315376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076687.315416:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076687.315487:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076687.443461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076687.443478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076687.443485:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076687.443498:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3676 00000100:00080000:3.0:1769076687.443512:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076687.443521:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076687.443566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617294720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076687.443608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076687.443615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076687.443619:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076687.443628:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3676 00000100:00080000:3.0:1769076687.443653:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769076687.443667:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617294720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076687.443668:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076687.443694:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076687.443728:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617294848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076687.443759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769076687.443766:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076687.443769:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076687.443775:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076687.443785:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3676 00000100:00080000:3.0:1769076687.443793:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076687.443800:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076687.443808:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076687.443830:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617294976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076687.443841:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076687/real 1769076687] req@ffff9b0352a1b100 x1855007617294720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076687.443843:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076687.443858:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076687.443865:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076687.443879:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3676 00000100:00080000:3.0:1769076687.443887:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076687.443892:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076687.443898:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617294720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076687.443902:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1769076687.443926:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769076687.443928:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617295104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076687.443944:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076687.443959:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617294720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076687.443961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076687.443994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076687.443997:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076687.444007:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3676 00000100:00080000:3.0:1769076687.444020:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076687.444025:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076687.444032:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076687.444051:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076687.444053:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617295232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076687.444108:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076687.444128:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076687/real 1769076687] req@ffff9b0352a1ad80 x1855007617294848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076687.444146:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617294848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076687.444157:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076687.444162:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076687.444166:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076687.444174:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076687.444194:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076687.444205:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076687/real 1769076687] req@ffff9b0352a1b480 x1855007617294976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076687.444219:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617294976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076687.444229:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076687.444264:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076687.444269:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617294976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076687.444277:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076687.444300:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076687.444314:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076687/real 1769076687] req@ffff9b0352a18a80 x1855007617295104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076687.444330:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617295104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076687.444340:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076687.444344:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076687.444348:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076687.444368:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076687.444386:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076687.444397:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076687/real 1769076687] req@ffff9b0352a1a680 x1855007617295232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076687.444417:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617295232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076687.444433:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076687.444440:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076687.444446:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076687.955483:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076688.339394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076688.339395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076688.339533:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076688.979390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076689.363427:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076689.363464:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076689.363506:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076690.003328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076690.387360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076690.387410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076690.387507:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076691.027392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076691.411419:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076691.411433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076691.411522:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076692.051415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076692.435387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076692.435486:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076692.435520:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769076692.563449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076692.563462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076692.563468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076692.563480:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3681 00000100:00080000:2.0:1769076692.563494:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076692.563504:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076692.563548:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617295360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076692.563588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076692.563593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076692.563596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076692.563601:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3681 00000100:00080000:2.0:1769076692.563605:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076692.563610:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076692.563620:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076692.563636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617295488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076692.563649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076692.563653:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076692.563655:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076692.563660:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3681 00000100:00080000:2.0:1769076692.563663:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076692.563668:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076692.563673:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076692.563685:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617295616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076692.563708:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769076692.563716:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617295360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076692.563716:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076692.563723:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076692.563736:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3681 00000100:00080000:2.0:1769076692.563741:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076692.563745:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076692.563752:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076692.563767:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617295744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076692.563780:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076692.563785:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076692.563788:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076692.563792:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3681 00000100:00080000:2.0:1769076692.563796:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076692.563801:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076692.563806:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769076692.563818:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076692.563825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617295872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076692.563911:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076692/real 1769076692] req@ffff9b03482c5500 x1855007617295360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076692.563949:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617295360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076692.563984:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076692.563994:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076692.564002:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617295360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076692.564017:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076692.564055:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076692.564074:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076692/real 1769076692] req@ffff9b03482c4380 x1855007617295488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076692.564090:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617295488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076692.564101:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076692.564105:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076692.564109:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076692.564139:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076692.564159:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076692.564171:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076692/real 1769076692] req@ffff9b03482c5180 x1855007617295616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076692.564185:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617295616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076692.564195:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076692.564198:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076692.564202:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076692.564210:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076692.564227:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076692.564289:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076692/real 1769076692] req@ffff9b03482c4a80 x1855007617295744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076692.564305:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617295744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076692.564316:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076692.564320:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076692.564324:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076692.564331:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076692.564354:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076692.564365:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076692/real 1769076692] req@ffff9b03482c6300 x1855007617295872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076692.564378:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617295872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076692.564387:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076692.564391:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076692.564395:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617295872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076693.075379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076693.459379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076693.459439:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076693.459531:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076694.099354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769076694.483471:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076694.483472:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076694.483499:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076695.123308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076695.507394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076695.507673:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076695.507754:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076696.147370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076696.531379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076696.531554:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076696.531594:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076697.171322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076697.555333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076697.555396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076697.555520:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076697.683328:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076697.683336:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076697.683340:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076697.683348:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3686 00000100:00080000:1.0:1769076697.683356:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076697.683373:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076697.683445:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617296000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076697.683469:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076697.683473:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076697.683475:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076697.683481:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3686 00000100:00080000:1.0:1769076697.683486:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076697.683491:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076697.683501:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076697.683511:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617296000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076697.683519:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617296128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076697.683530:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076697.683533:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076697.683534:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076697.683538:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3686 00000100:00080000:1.0:1769076697.683541:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076697.683544:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076697.683567:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076697.683577:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617296256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769076697.683583:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076697.683591:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076697.683594:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076697.683596:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076697.683600:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3686 00000100:00080000:1.0:1769076697.683603:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076697.683606:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076697.683610:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076697.683624:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076697/real 1769076697] req@ffff9b0343def100 x1855007617296000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076697.683625:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617296384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076697.683652:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617296000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076697.683661:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076697.683671:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076697.683675:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076697.683676:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076697.683684:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3686 00000100:00080000:0.0:1769076697.683685:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769076697.683708:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076697.683715:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076697.683733:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617296000:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769076697.683735:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076697.683752:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617296512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076697.683763:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076697.683791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076697.683846:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076697/real 1769076697] req@ffff9b0343dedf80 x1855007617296128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076697.683871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617296128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076697.683883:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076697.683886:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076697.683888:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076697.683893:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076697.683909:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076697.683917:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076697/real 1769076697] req@ffff9b0343ded880 x1855007617296256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076697.683932:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617296256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076697.683942:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076697.683944:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076697.683946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076697.683950:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076697.683962:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076697.683968:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076697/real 1769076697] req@ffff9b0343def800 x1855007617296384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076697.683976:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617296384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076697.683981:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076697.683983:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076697.683985:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076697.683996:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076697.684005:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076697.684011:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076697/real 1769076697] req@ffff9b0343deea00 x1855007617296512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076697.684019:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617296512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076697.684023:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076697.684025:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076697.684027:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076698.195335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769076698.579490:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076698.579502:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076698.579536:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076699.219322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076699.603340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076699.603349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076699.603443:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076700.243354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076700.627368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076700.627384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076700.627548:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076701.267456:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076701.651393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076701.651401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076701.651505:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076702.291373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076702.675355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076702.675415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076702.675494:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076702.803345:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076702.803359:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076702.803364:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076702.803391:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3691 00000100:00080000:1.0:1769076702.803407:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076702.803455:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076702.803512:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617296640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076702.803546:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076702.803551:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076702.803554:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076702.803560:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3691 00000100:00080000:1.0:1769076702.803565:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076702.803570:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076702.803581:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076702.803605:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617296768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076702.803618:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076702.803623:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076702.803625:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076702.803630:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3691 00000100:00080000:1.0:1769076702.803634:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076702.803640:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076702.803653:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617296640:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769076702.803655:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076702.803674:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617296896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076702.803683:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076702.803686:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076702.803688:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076702.803691:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3691 00000100:00080000:1.0:1769076702.803694:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076702.803697:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076702.803702:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076702.803730:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617297024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076702.803747:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076702.803750:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076702.803751:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076702.803754:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3691 00000100:00080000:1.0:1769076702.803759:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076702.803762:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076702.803766:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769076702.803774:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076702.803782:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617297152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769076702.803826:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076702/real 1769076702] req@ffff9b0343deed80 x1855007617296640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076702.803843:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617296640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076702.803852:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076702.803856:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076702.803860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617296640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076702.803866:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076702.804058:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076702.804095:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076702/real 1769076702] req@ffff9b0343deca80 x1855007617296768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076702.804117:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617296768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076702.804130:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076702.804135:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076702.804140:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076702.804174:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769076702.804204:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076702.804219:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076702/real 1769076702] req@ffff9b0343dece00 x1855007617296896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076702.804288:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617296896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076702.804302:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076702.804307:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076702.804312:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617296896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076702.804320:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076702.804346:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076702.804358:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076702/real 1769076702] req@ffff9b0343defb80 x1855007617297024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076702.804370:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617297024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076702.804396:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076702.804400:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076702.804403:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076702.804410:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769076702.804430:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076702.804456:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076702/real 1769076702] req@ffff9b0343dee680 x1855007617297152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076702.804474:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617297152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076702.804481:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076702.804484:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769076702.804486:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076703.315386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076703.699353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076703.699405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076703.699425:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076704.339395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076704.723404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076704.723404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076704.723418:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076705.363410:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076705.747361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076705.747388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076705.747478:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076706.387455:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076706.771372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076706.771416:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076706.774137:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076707.411351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076707.795444:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076707.795449:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076707.795472:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076707.923414:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076707.923437:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076707.923442:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076707.923453:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3697 00000100:00080000:3.0:1769076707.923467:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076707.923475:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076707.923535:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617297280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076707.923567:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076707.923571:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076707.923573:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076707.923577:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3697 00000100:00080000:3.0:1769076707.923580:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076707.923583:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076707.923594:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076707.923606:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617297408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076707.923611:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617297280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076707.923651:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076707.923658:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076707.923661:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076707.923667:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3697 00000100:00080000:3.0:1769076707.923674:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076707.923680:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076707.923686:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076707.923703:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617297536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076707.923713:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076707.923716:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076707.923718:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076707.923724:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3697 00000800:00000100:1.0:1769076707.923731:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076707.923732:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076707.923739:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076707.923746:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076707.923761:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617297664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076707.923768:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076707.923771:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076707.923774:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076707.923782:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3697 00000100:00080000:3.0:1769076707.923786:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076707.923789:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076707.923793:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076707/real 1769076707] req@ffff9b0352a18700 x1855007617297280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076707.923795:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076707.923813:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617297280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076707.923822:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076707.923826:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076707.923834:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617297280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076707.923834:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617297792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076707.923866:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076707.923884:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076707.923892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076707/real 1769076707] req@ffff9b0352a18e00 x1855007617297408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076707.923901:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617297408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076707.923907:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076707.923909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076707.923911:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076707.923923:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076707.925174:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076707.925184:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076707/real 1769076707] req@ffff9b0352a1a300 x1855007617297536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076707.925192:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617297536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076707.925199:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076707.925202:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076707.925204:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076707.925210:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076707.925222:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076707.925228:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076707/real 1769076707] req@ffff9b0352a19f80 x1855007617297664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076707.925252:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617297664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076707.925258:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076707.925261:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076707.925264:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076707.925269:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076707.925294:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076707.925302:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076707/real 1769076707] req@ffff9b0352a1aa00 x1855007617297792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076707.925310:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617297792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076707.925318:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076707.925321:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076707.925324:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617297792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076708.435443:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076708.819301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076708.819335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076708.819367:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076709.459362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076709.843303:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076709.843321:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076709.843337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076710.483329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076710.867287:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076710.867340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076710.867399:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076711.507383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076711.891341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076711.891377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076711.891469:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076712.531382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076712.915337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076712.915364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076712.915442:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076713.043334:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076713.043347:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076713.043359:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076713.043371:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3702 00000100:00080000:3.0:1769076713.043382:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076713.043389:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076713.043437:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617297920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076713.043467:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076713.043471:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076713.043481:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076713.043485:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3702 00000100:00080000:3.0:1769076713.043489:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076713.043494:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076713.043506:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076713.043520:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617298048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076713.043527:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617297920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076713.043532:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076713.043550:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076713.043552:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076713.043556:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3702 00000100:00080000:3.0:1769076713.043560:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076713.043564:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076713.043570:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076713.043581:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617298176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076713.043589:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076713.043593:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076713.043595:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076713.043598:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3702 00000100:00080000:3.0:1769076713.043602:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076713.043605:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076713.043610:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076713.043618:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617298304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076713.043626:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076713.043629:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076713.043631:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076713.043643:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3702 00000100:00080000:3.0:1769076713.043647:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076713.043651:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076713.043657:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076713.043667:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617298432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076713.043681:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076713.043714:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076713.043739:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076713.043758:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076713.555293:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076713.939309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076713.939313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076713.939353:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076714.579308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076714.963354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076714.963358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076714.963397:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076715.603279:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076715.987405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076715.987412:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076715.987613:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769076716.115715:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769076716.115811:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769076716.115852:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076716.115904:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076716.115916:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076716.115928:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076716.115941:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076716.115958:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076716] req@ffff9b0352a19f80 x1855007617298048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076716.115988:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617298048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076716.116004:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076716.116010:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076716.116016:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076716.116026:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076716] req@ffff9b0352a1a300 x1855007617298176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076716.116040:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617298176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076716.116049:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076716.116052:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076716.116056:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076716.116064:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076716] req@ffff9b0352a18e00 x1855007617298304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076716.116077:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617298304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076716.116087:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076716.116090:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076716.116094:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076716.116101:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076716] req@ffff9b0352a1aa00 x1855007617297920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076716.116114:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617297920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076716.116122:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076716.116125:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076716.116129:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617297920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076716.116176:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076713/real 1769076716] req@ffff9b0352a18700 x1855007617298432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076716.116189:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617298432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076716.116199:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076716.116203:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076716.116206:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076716.627400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076717.011416:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076717.011417:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076717.011608:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076717.651359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076718.035418:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076718.035428:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076718.035577:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076718.163407:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076718.163427:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076718.163434:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076718.163496:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3707 00000100:00080000:3.0:1769076718.163512:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076718.163523:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076718.163574:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617298560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076718.163613:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076718.163618:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076718.163620:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076718.163625:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3707 00000100:00080000:3.0:1769076718.163629:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076718.163632:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076718.163645:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076718.163659:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617298688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076718.163673:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076718.163677:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076718.163679:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076718.163683:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3707 00000100:00080000:3.0:1769076718.163686:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076718.163690:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076718.163695:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076718.163701:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617298560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076718.163724:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617298816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076718.163737:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076718.163742:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076718.163744:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076718.163749:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3707 00000100:00080000:3.0:1769076718.163753:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076718.163757:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076718.163762:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076718.163775:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617298944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076718.163785:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076718.163789:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076718.163792:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076718.163797:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3707 00000100:00080000:3.0:1769076718.163802:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076718.163806:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769076718.163813:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769076718.163815:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076718.163842:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617299072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076718.163879:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076718/real 1769076718] req@ffff9b0352a1a680 x1855007617298560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076718.163904:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617298560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076718.163918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076718.163923:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076718.163928:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617298560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076718.163936:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076718.163974:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076718.163987:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076718/real 1769076718] req@ffff9b0352a18a80 x1855007617298688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076718.164000:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617298688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076718.164010:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076718.164013:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076718.164017:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076718.164057:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076718.164087:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076718.164105:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076718/real 1769076718] req@ffff9b0352a1b480 x1855007617298816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076718.164125:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617298816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076718.164138:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076718.164143:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076718.164148:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076718.164157:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076718.164189:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076718.164207:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076718/real 1769076718] req@ffff9b0352a1ad80 x1855007617298944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076718.164225:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617298944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076718.164310:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076718.164316:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076718.164322:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617298944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076718.164332:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076718.164381:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076718.164398:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076718/real 1769076718] req@ffff9b0352a1b100 x1855007617299072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076718.164418:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617299072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076718.164443:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076718.164472:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076718.164480:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076718.675402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076719.059365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076719.059366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076719.059536:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076719.699387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076720.083369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076720.083371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076720.083414:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076720.723582:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076721.107459:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076721.107485:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076721.107610:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076721.747368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076722.131352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076722.131377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076722.131546:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076722.771299:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076723.155399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076723.155423:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076723.155458:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076723.283431:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076723.283447:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076723.283454:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076723.283469:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3712 00000100:00080000:0.0:1769076723.283485:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076723.283495:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076723.283553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617299200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076723.283599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076723.283604:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076723.283606:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076723.283611:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3712 00000100:00080000:0.0:1769076723.283615:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076723.283619:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076723.283632:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076723.283651:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617299328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076723.283666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076723.283670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076723.283671:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076723.283679:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3712 00000100:00100000:1.0:1769076723.283683:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617299200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076723.283684:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076723.283691:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076723.283700:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076723.283715:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617299456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076723.283725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076723.283728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076723.283731:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076723.283736:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3712 00000100:00080000:0.0:1769076723.283741:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076723.283745:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076723.283750:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076723.283762:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617299584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076723.283771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076723.283775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076723.283777:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076723.283782:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3712 00000100:00080000:0.0:1769076723.283796:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076723.283800:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076723.283806:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076723.283811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769076723.283825:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617299712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076723.283892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076723/real 1769076723] req@ffff9b0365a3ad80 x1855007617299200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076723.283914:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617299200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076723.283926:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076723.283932:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076723.283936:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617299200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076723.283967:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076723.283990:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076723.284002:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076723/real 1769076723] req@ffff9b0365a39180 x1855007617299328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076723.284014:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617299328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076723.284035:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076723.284038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076723.284041:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076723.284047:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076723.284064:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076723.284073:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076723/real 1769076723] req@ffff9b0365a39500 x1855007617299456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076723.284084:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617299456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076723.284092:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076723.284095:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076723.284098:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076723.284103:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076723.284117:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076723.284131:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076723/real 1769076723] req@ffff9b0365a3b100 x1855007617299584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076723.284148:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617299584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076723.284160:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076723.284165:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076723.284171:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076723.284180:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076723.284210:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076723.284224:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076723/real 1769076723] req@ffff9b0365a3bb80 x1855007617299712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076723.284287:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617299712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076723.284301:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076723.284307:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076723.284312:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076723.795392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076724.179483:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076724.179505:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076724.179553:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076724.819507:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076725.203303:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076725.203327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076725.203348:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076725.843303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076726.227376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076726.227380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076726.227459:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076726.867341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076727.251440:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076727.251444:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076727.251560:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076727.891397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076728.275452:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076728.275462:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076728.275597:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076728.403446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076728.403468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076728.403494:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076728.403513:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3717 00000100:00080000:3.0:1769076728.403532:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076728.403546:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076728.403638:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617299840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076728.403688:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076728.403695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076728.403700:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076728.403724:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3717 00000100:00080000:3.0:1769076728.403731:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076728.403739:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076728.403758:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076728.403783:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617299968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076728.403787:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617299840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076728.403803:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076728.403810:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076728.403813:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076728.403820:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3717 00000100:00080000:3.0:1769076728.403826:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076728.403833:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076728.403841:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076728.403857:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617300096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076728.403873:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076728.403878:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076728.403880:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076728.403888:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3717 00000100:00080000:3.0:1769076728.403894:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076728.403900:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076728.403909:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076728.403920:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076728.403924:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617300224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076728.403939:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076728.403944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076728.403947:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076728.403954:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3717 00000100:00080000:3.0:1769076728.403960:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076728.403966:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076728.403989:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076728.403995:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076728/real 1769076728] req@ffff9b0352a18380 x1855007617299840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076728.404008:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617300352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076728.404030:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617299840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076728.404048:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076728.404055:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076728.404064:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617299840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076728.404109:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076728.404157:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076728.404201:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076728/real 1769076728] req@ffff9b0352a1b800 x1855007617299968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076728.404228:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617299968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076728.404340:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076728.404348:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076728.404355:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617299968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076728.404368:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076728.404412:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076728.404434:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076728/real 1769076728] req@ffff9b0352a19500 x1855007617300096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076728.404475:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617300096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076728.404491:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076728.404497:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076728.404503:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076728.404515:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076728.404557:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076728.404581:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076728/real 1769076728] req@ffff9b0352a18000 x1855007617300224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076728.404606:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617300224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076728.404617:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076728.404621:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076728.404626:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076728.404634:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076728.404661:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076728.404674:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076728/real 1769076728] req@ffff9b0352a19c00 x1855007617300352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076728.404687:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617300352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076728.404698:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076728.404701:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076728.404705:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076728.915457:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076729.299385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076729.299417:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076729.299506:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076729.939453:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076730.323338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076730.323369:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076730.323375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076730.963376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076731.347322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076731.347399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076731.347417:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076731.987342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076732.371378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076732.371378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076732.371451:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076733.011324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076733.395349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076733.395350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076733.395423:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076733.523350:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076733.523361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076733.523365:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076733.523374:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3722 00000100:00080000:3.0:1769076733.523382:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076733.523388:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076733.523422:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617300480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076733.523445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076733.523448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076733.523450:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076733.523453:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3722 00000100:00080000:3.0:1769076733.523455:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076733.523458:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076733.523465:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076733.523478:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617300608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076733.523480:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617300480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076733.523495:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076733.523497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076733.523499:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076733.523502:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3722 00000100:00080000:3.0:1769076733.523504:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076733.523506:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076733.523509:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076733.523517:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617300736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076733.523524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076733.523526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076733.523528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076733.523530:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3722 00000100:00080000:3.0:1769076733.523532:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076733.523534:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076733.523537:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076733.523544:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617300864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076733.523551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769076733.523555:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076733.523557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076733.523561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076733.523565:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3722 00000100:00080000:3.0:1769076733.523570:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076733.523583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076733.523589:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076733.523604:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617300992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076733.523610:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076733/real 1769076733] req@ffff9b0352a19c00 x1855007617300480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076733.523637:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617300480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076733.523647:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076733.523652:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076733.523656:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617300480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076733.523679:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076733.523701:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076733.523714:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076733/real 1769076733] req@ffff9b0352a18000 x1855007617300608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076733.523727:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617300608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076733.523734:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076733.523736:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076733.523739:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076733.523743:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076733.523757:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076733.523764:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076733/real 1769076733] req@ffff9b0352a19500 x1855007617300736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076733.523772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617300736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076733.523782:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076733.523784:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076733.523786:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076733.523790:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076733.523800:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076733.523805:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076733/real 1769076733] req@ffff9b0352a1b800 x1855007617300864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076733.523811:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617300864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076733.523816:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076733.523818:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076733.523820:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076733.523823:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076733.523831:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076733.523836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076733/real 1769076733] req@ffff9b0352a18380 x1855007617300992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076733.523844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617300992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076733.523848:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076733.523850:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076733.523853:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617300992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076734.035372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076734.419347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076734.419369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076734.419438:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076735.059319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076735.443320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076735.443332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076735.443425:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076736.083386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076736.467354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076736.467354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076736.467456:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076737.107391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076737.491368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076737.491394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076737.491612:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076738.131374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076738.515319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076738.515359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076738.515394:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076738.643343:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076738.643354:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076738.643359:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076738.643371:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3727 00000100:00080000:2.0:1769076738.643381:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076738.643389:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076738.643440:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617301120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076738.643464:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076738.643467:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076738.643469:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076738.643472:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3727 00000100:00080000:2.0:1769076738.643474:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076738.643477:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076738.643485:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076738.643495:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617301248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076738.643509:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076738.643513:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617301120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076738.643513:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076738.643515:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076738.643522:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3727 00000100:00080000:2.0:1769076738.643525:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076738.643528:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076738.643532:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076738.643542:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617301376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076738.643548:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076738.643551:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076738.643552:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076738.643555:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3727 00000100:00080000:2.0:1769076738.643558:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076738.643560:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076738.643564:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076738.643570:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617301504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076738.643576:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076738.643578:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076738.643580:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076738.643582:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3727 00000100:00080000:2.0:1769076738.643586:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076738.643590:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076738.643591:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076738.643597:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076738.643606:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617301632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076738.643639:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076738/real 1769076738] req@ffff9b03482c6d80 x1855007617301120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076738.643655:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617301120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076738.643665:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076738.643668:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076738.643671:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617301120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076738.643695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617301248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076738.643710:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076738.643718:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076738/real 1769076738] req@ffff9b03482c4700 x1855007617301248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076738.643727:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617301248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076738.643732:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076738.643734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076738.643736:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617301248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076738.643741:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617301376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076738.643751:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076738.643757:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076738/real 1769076738] req@ffff9b03482c7100 x1855007617301376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076738.643765:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617301376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076738.643770:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076738.643772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076738.643774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617301376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076738.643778:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617301504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076738.643788:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076738.643794:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076738/real 1769076738] req@ffff9b03482c4000 x1855007617301504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076738.643800:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617301504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076738.643805:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076738.643807:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076738.643810:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617301504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076738.643813:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617301632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076738.643822:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076738.643827:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076738/real 1769076738] req@ffff9b03482c6a00 x1855007617301632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076738.643835:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617301632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076738.643840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076738.643842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076738.643844:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617301632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076739.155344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769076739.539310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076739.539435:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076739.539491:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076740.179408:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076740.563448:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076740.563449:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076740.563496:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076741.203413:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076741.587306:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076741.587392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076741.587473:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076742.227367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076742.611370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076742.611466:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076742.611472:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076743.251419:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076743.635324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076743.635361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076743.635471:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076743.763496:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076743.763508:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076743.763513:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076743.763525:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3732 00000100:00080000:1.0:1769076743.763537:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076743.763545:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076743.763586:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617301760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076743.763616:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076743.763621:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076743.763623:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076743.763646:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3732 00000100:00080000:1.0:1769076743.763654:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076743.763661:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076743.763673:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076743.763685:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617301760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076743.763700:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617301888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076743.763724:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076743.763731:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076743.763733:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076743.763740:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3732 00000100:00080000:1.0:1769076743.763746:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076743.763750:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076743.763755:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076743.763758:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076743.763776:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617302016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076743.763788:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076743.763790:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076743.763792:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076743.763799:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3732 00000100:00080000:1.0:1769076743.763805:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076743.763809:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076743.763814:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076743.763815:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076743/real 1769076743] req@ffff9b0343ded500 x1855007617301760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076743.763824:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617302144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076743.763833:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617301760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076743.763835:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076743.763842:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076743.763846:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076743.763854:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3732 00000100:00080000:2.0:1769076743.763855:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076743.763857:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076743.763863:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769076743.763864:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076743.763871:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076743.763871:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617301760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076743.763884:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617302272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076743.763929:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617301888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076743.763975:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076743.763993:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076743/real 1769076743] req@ffff9b0343def480 x1855007617301888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076743.764009:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617301888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076743.764058:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076743.764062:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076743.764065:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617301888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076743.764070:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076743.764089:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076743.764099:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076743/real 1769076743] req@ffff9b0343dee680 x1855007617302016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076743.764108:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617302016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076743.764114:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076743.764117:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076743.764120:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076743.764126:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076743.764142:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076743.764153:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076743/real 1769076743] req@ffff9b0343defb80 x1855007617302144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076743.764165:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617302144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076743.764174:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076743.764183:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076743.764187:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076743.764192:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076743.764208:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076743.764219:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076743/real 1769076743] req@ffff9b0343dece00 x1855007617302272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076743.764231:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617302272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076743.764239:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076743.764242:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076743.764246:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076744.275390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076744.659401:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076744.659409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076744.659458:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076745.299323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076745.683333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076745.683333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076745.683362:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076746.323339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076746.707318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076746.707327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076746.707391:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076747.347343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076747.731313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076747.731313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076747.731405:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076748.371354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076748.755316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076748.755353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076748.755391:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076748.883317:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076748.883327:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076748.883331:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076748.883340:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3738 00000100:00080000:1.0:1769076748.883350:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076748.883357:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076748.883390:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617302400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076748.883414:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076748.883417:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076748.883418:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076748.883421:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3738 00000100:00080000:1.0:1769076748.883423:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076748.883426:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076748.883432:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076748.883440:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617302528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076748.883447:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076748.883450:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076748.883451:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076748.883453:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3738 00000100:00080000:1.0:1769076748.883455:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076748.883457:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076748.883460:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076748.883466:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617302656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076748.883472:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076748.883474:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076748.883475:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076748.883478:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3738 00000100:00080000:1.0:1769076748.883481:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076748.883483:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076748.883487:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076748.883489:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617302400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076748.883497:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617302784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076748.883504:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076748.883507:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076748.883509:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076748.883511:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3738 00000100:00080000:1.0:1769076748.883515:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076748.883517:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076748.883521:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076748.883529:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617302912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076748.883552:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076748.883601:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076748/real 1769076748] req@ffff9b0343deca80 x1855007617302400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076748.883622:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617302400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076748.883635:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076748.883639:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076748.883643:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617302400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076748.883650:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076748.883669:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076748.883678:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076748/real 1769076748] req@ffff9b0343deed80 x1855007617302528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076748.883686:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617302528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076748.883692:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076748.883694:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076748.883696:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076748.883700:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076748.883711:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076748.883716:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076748/real 1769076748] req@ffff9b0343deea00 x1855007617302656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076748.883723:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617302656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076748.883729:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076748.883730:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076748.883732:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076748.883753:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076748.883763:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076748.883769:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076748/real 1769076748] req@ffff9b0343def800 x1855007617302784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076748.883776:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617302784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076748.883781:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076748.883783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076748.883784:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076748.883788:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076748.883796:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076748.883808:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076748/real 1769076748] req@ffff9b0343ded880 x1855007617302912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076748.883816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617302912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076748.883821:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076748.883823:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076748.883825:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617302912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076749.395342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076749.779297:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076749.779375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076749.779437:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076750.419331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076750.803321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076750.803321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076750.803341:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076751.443319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076751.827349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076751.827349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076751.827418:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076752.467343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076752.851361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076752.851361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076752.851458:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076753.491341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076753.875378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076753.875378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076753.875475:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076754.003369:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076754.003380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076754.003384:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076754.003395:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3743 00000100:00080000:0.0:1769076754.003406:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076754.003413:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076754.003461:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617303040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076754.003501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076754.003504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076754.003506:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076754.003510:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3743 00000100:00080000:0.0:1769076754.003513:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076754.003518:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076754.003528:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076754.003545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617303168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076754.003554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076754.003557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076754.003559:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076754.003562:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3743 00000100:00080000:0.0:1769076754.003565:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076754.003568:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076754.003571:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076754.003581:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617303040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076754.003583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617303296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076754.003606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076754.003610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076754.003611:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076754.003617:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3743 00000100:00080000:0.0:1769076754.003620:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076754.003623:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076754.003629:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076754.003643:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617303424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076754.003656:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076754.003660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076754.003663:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769076754.003671:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076754.003680:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3743 00000100:00080000:0.0:1769076754.003689:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076754.003695:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076754.003701:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076754.003715:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617303552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076754.003732:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9b0365a3aa00 x1855007617303040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076754.003751:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617303040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076754.003760:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076754.003765:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076754.003768:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617303040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076754.003775:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076754.003791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076754.003799:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9b0365a3b480 x1855007617303168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076754.003806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617303168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076754.003812:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076754.003814:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076754.003816:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076754.003837:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076754.003861:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076754.003869:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9b0365a38e00 x1855007617303296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076754.003877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617303296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076754.003882:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076754.003884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076754.003887:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076754.003891:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076754.003903:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076754.003909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9b0365a39c00 x1855007617303424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076754.003916:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617303424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076754.003921:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076754.003923:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076754.003925:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076754.003929:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076754.003939:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076754.003944:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076754/real 1769076754] req@ffff9b0365a38a80 x1855007617303552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076754.003951:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617303552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076754.003961:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076754.003963:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076754.003965:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076754.515333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076754.899329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076754.899329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076754.899376:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076755.539331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076755.923422:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076755.923428:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076755.923467:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076756.563349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076756.947352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076756.947403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076756.947429:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076757.587356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076757.971379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076757.971380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076757.971451:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076758.611367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076758.995330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076758.995330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076758.995393:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076759.123330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076759.123339:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076759.123343:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076759.123351:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3748 00000100:00080000:0.0:1769076759.123361:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076759.123367:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076759.123401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617303680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076759.123425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076759.123428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076759.123430:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076759.123433:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3748 00000100:00080000:0.0:1769076759.123435:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076759.123437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076759.123444:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076759.123453:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617303808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076759.123460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076759.123462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076759.123464:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076759.123467:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3748 00000100:00080000:0.0:1769076759.123469:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076759.123471:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076759.123475:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076759.123479:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617303680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076759.123483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617303936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076759.123494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076759.123496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076759.123498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076759.123501:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3748 00000100:00080000:0.0:1769076759.123504:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076759.123506:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076759.123509:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076759.123516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617304064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076759.123522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076759.123525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076759.123526:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076759.123529:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3748 00000100:00080000:0.0:1769076759.123532:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076759.123534:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076759.123536:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076759.123543:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617304192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076759.123556:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076759.123604:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076759/real 1769076759] req@ffff9b0365a38a80 x1855007617303680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076759.123624:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617303680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076759.123637:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076759.123642:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076759.123646:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617303680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076759.123654:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076759.123675:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076759.123686:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076759/real 1769076759] req@ffff9b0365a39c00 x1855007617303808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076759.123699:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617303808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076759.123709:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076759.123713:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076759.123716:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076759.123740:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076759.123759:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076759.123786:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076759/real 1769076759] req@ffff9b0365a38e00 x1855007617303936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076759.123798:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617303936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076759.123807:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076759.123810:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076759.123813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617303936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076759.123819:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076759.123839:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076759.123848:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076759/real 1769076759] req@ffff9b0365a3b480 x1855007617304064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076759.123859:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617304064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076759.123868:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076759.123871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076759.123875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076759.123881:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076759.123897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076759.123907:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076759/real 1769076759] req@ffff9b0365a3aa00 x1855007617304192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076759.123919:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617304192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076759.123928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076759.123931:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076759.123935:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076759.635320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076760.019359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076760.019483:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076760.020115:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076760.659319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076761.043359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076761.043417:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076761.043488:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076761.683309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076762.067379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076762.067567:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076762.067580:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076762.707313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769076763.091376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076763.091403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076763.091486:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076763.731330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076764.115384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076764.115407:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076764.115409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:1.0:1769076764.243398:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076764.243409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076764.243414:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076764.243424:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3753 00000100:00080000:1.0:1769076764.243435:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076764.243442:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076764.243480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617304320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076764.243509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076764.243514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076764.243516:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076764.243520:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3753 00000100:00080000:1.0:1769076764.243523:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076764.243528:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076764.243536:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076764.243547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617304448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076764.243556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076764.243559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076764.243561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076764.243564:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3753 00000100:00080000:1.0:1769076764.243567:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076764.243570:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076764.243574:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076764.243583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617304576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076764.243592:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076764.243595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076764.243597:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076764.243604:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3753 00000100:00100000:2.0:1769076764.243605:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617304320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076764.243608:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076764.243617:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076764.243622:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076764.243633:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617304704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076764.243641:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076764.243646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076764.243647:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076764.243651:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3753 00000100:00080000:1.0:1769076764.243654:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076764.243658:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076764.243662:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076764.243670:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617304832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076764.243717:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076764.243759:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076764/real 1769076764] req@ffff9b0343dedf80 x1855007617304320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076764.243776:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617304320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076764.243786:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076764.243791:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076764.243794:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617304320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076764.243803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076764.243823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076764.243834:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076764/real 1769076764] req@ffff9b0343def100 x1855007617304448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076764.243844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617304448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076764.243852:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076764.243855:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076764.243858:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076764.243881:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076764.243896:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076764.243904:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076764/real 1769076764] req@ffff9b0343dee300 x1855007617304576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076764.243914:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617304576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076764.243921:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076764.243924:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076764.243926:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076764.243932:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076764.243945:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076764.243953:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076764/real 1769076764] req@ffff9b0343dec380 x1855007617304704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076764.243964:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617304704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076764.243971:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076764.243973:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076764.243976:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076764.243980:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076764.243992:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076764.243999:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076764/real 1769076764] req@ffff9b0343ded180 x1855007617304832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076764.244008:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617304832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076764.244015:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076764.244018:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076764.244020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617304832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076764.755358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076765.139423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076765.139735:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076765.139880:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076765.779422:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076766.163400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076766.163401:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076766.163432:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076766.803358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076767.187308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076767.187375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076767.187419:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076767.827318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076768.211327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076768.211398:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076768.211410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076768.851336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076769.235316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076769.235348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076769.235429:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076769.363342:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076769.363354:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076769.363359:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076769.363370:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3758 00000100:00080000:1.0:1769076769.363382:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076769.363390:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076769.363427:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617304960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076769.363455:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076769.363459:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076769.363460:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076769.363464:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3758 00000100:00080000:1.0:1769076769.363467:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076769.363470:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076769.363480:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076769.363498:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617305088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076769.363515:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076769.363515:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617304960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076769.363525:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076769.363531:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076769.363540:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3758 00000100:00080000:1.0:1769076769.363545:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076769.363552:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076769.363558:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076769.363574:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617305216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076769.363583:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076769.363586:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076769.363588:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076769.363608:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3758 00000100:00080000:1.0:1769076769.363612:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076769.363616:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076769.363621:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769076769.363623:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076769.363636:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617305344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076769.363647:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076769.363650:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076769.363652:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076769.363656:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3758 00000100:00080000:1.0:1769076769.363660:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076769.363664:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076769.363669:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076769.363680:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617305472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076769.363683:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076769/real 1769076769] req@ffff9b0343dec000 x1855007617304960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076769.363712:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617304960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076769.363722:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076769.363726:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076769.363730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617304960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076769.363756:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076769.363786:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076769.363803:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076769/real 1769076769] req@ffff9b0343dec700 x1855007617305088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076769.363819:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617305088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076769.363827:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076769.363830:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076769.363832:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076769.363838:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076769.363856:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076769.363865:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076769/real 1769076769] req@ffff9b0343ded180 x1855007617305216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076769.363880:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617305216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076769.363886:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076769.363889:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076769.363891:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076769.363896:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076769.363909:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076769.363915:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076769/real 1769076769] req@ffff9b0343dec380 x1855007617305344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076769.363925:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617305344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076769.363931:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076769.363933:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076769.363935:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076769.363939:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076769.363950:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076769.363956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076769/real 1769076769] req@ffff9b0343dee300 x1855007617305472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076769.363964:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617305472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076769.363975:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076769.363978:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076769.363980:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076769.875415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076770.259359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076770.259360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076770.259473:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076770.899331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076771.283350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076771.283394:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076771.283418:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076771.923336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076772.307362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076772.307382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076772.307438:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076772.947347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076773.331333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076773.331366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076773.331408:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076773.971350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076774.355320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076774.355320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076774.355478:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076774.483361:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076774.483371:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076774.483375:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076774.483384:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3763 00000100:00080000:3.0:1769076774.483393:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076774.483400:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076774.483432:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617305600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076774.483455:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076774.483460:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076774.483462:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076774.483467:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3763 00000100:00080000:3.0:1769076774.483471:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076774.483475:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076774.483484:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076774.483497:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617305728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076774.483520:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617305600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076774.483536:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076774.483539:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076774.483541:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076774.483546:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3763 00000100:00080000:3.0:1769076774.483550:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076774.483553:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076774.483559:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076774.483569:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617305856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076774.483578:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076774.483581:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1769076774.483582:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076774.483583:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076774.483586:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3763 00000100:00080000:3.0:1769076774.483590:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076774.483593:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076774.483597:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076774.483606:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617305984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076774.483614:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076774.483617:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076774.483620:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076774.483622:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076774/real 1769076774] req@ffff9b0352a1b100 x1855007617305600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076774.483624:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3763 00000100:00080000:3.0:1769076774.483627:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076774.483631:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076774.483635:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076774.483640:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617305600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076774.483644:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617306112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076774.483651:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076774.483655:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076774.483659:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617305600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076774.483683:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076774.483705:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076774.483715:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076774/real 1769076774] req@ffff9b0352a1ad80 x1855007617305728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076774.483728:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617305728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076774.483734:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076774.483736:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076774.483738:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076774.483742:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076774.483753:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076774.483758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076774/real 1769076774] req@ffff9b0352a1b480 x1855007617305856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076774.483766:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617305856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076774.483787:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076774.483789:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076774.483791:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076774.483795:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076774.483808:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076774.483815:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076774/real 1769076774] req@ffff9b0352a18a80 x1855007617305984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076774.483823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617305984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076774.483834:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076774.483836:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076774.483838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617305984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076774.483842:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617306112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076774.483868:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076774.483874:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076774/real 1769076774] req@ffff9b0352a1a680 x1855007617306112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076774.483881:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617306112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076774.483885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076774.483887:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076774.483889:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617306112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076774.995359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076775.379341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076775.379351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076775.379429:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076776.019430:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076776.403341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076776.403347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076776.403372:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076777.043371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076777.427367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076777.427421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076777.427496:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076778.067371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076778.451332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076778.451434:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076778.451434:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076779.091398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076779.475391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076779.475395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076779.475468:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076779.603456:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076779.603497:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076779.603504:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076779.603521:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3768 00000100:00080000:2.0:1769076779.603536:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076779.603548:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076779.603613:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617306240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076779.603691:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076779.603715:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076779.603717:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076779.603722:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3768 00000100:00080000:2.0:1769076779.603727:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076779.603732:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076779.603743:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076779.603758:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617306368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076779.603769:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076779.603774:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076779.603776:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076779.603780:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3768 00000100:00080000:2.0:1769076779.603783:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076779.603787:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076779.603803:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076779.603828:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617306496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076779.603842:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617306240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076779.603848:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076779.603863:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076779.603865:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076779.603882:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3768 00000100:00080000:2.0:1769076779.603887:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076779.603892:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076779.603898:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076779.603912:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617306624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076779.603921:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076779.603926:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076779.603928:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076779.603933:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3768 00000100:00080000:2.0:1769076779.603936:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076779.603942:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076779.603947:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076779.603956:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617306752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076779.604053:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617306368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076779.604111:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617306496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076779.604185:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617306624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076779.604359:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617306752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076780.115377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076780.499361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076780.499364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076780.499583:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076781.139297:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076781.523365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076781.523371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076781.523446:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076782.163298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076782.547292:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076782.547300:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076782.547411:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769076782.675712:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769076782.675788:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769076782.675816:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076782.675886:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076782.675916:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076782.675927:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769076782.675940:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076782.676028:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076779/real 1769076782] req@ffff9b03482c6680 x1855007617306496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076782.676075:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617306496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076782.676097:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076782.676104:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076782.676109:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617306496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076782.676118:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076779/real 1769076782] req@ffff9b03482c5880 x1855007617306624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076782.676131:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617306624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076782.676139:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076782.676143:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076782.676147:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617306624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076782.676154:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076779/real 1769076782] req@ffff9b03482c7b80 x1855007617306752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076782.676167:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617306752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076782.676176:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076782.676180:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076782.676191:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617306752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076782.676200:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076779/real 1769076782] req@ffff9b03482c5f80 x1855007617306240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076782.676216:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617306240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076782.676225:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076782.676229:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076782.676263:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617306240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076782.676271:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076779/real 1769076782] req@ffff9b03482c5c00 x1855007617306368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076782.676285:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617306368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076782.676295:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076782.676298:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076782.676301:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617306368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076783.187349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076783.571406:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076783.571421:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076783.571513:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076784.212062:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076784.595366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076784.595385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076784.595535:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076784.723425:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076784.723440:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076784.723446:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076784.723460:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3773 00000100:00080000:2.0:1769076784.723473:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076784.723482:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076784.723528:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617306880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076784.723566:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076784.723571:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076784.723573:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076784.723579:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3773 00000100:00080000:2.0:1769076784.723583:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076784.723587:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076784.723601:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076784.723627:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617307008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076784.723647:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617306880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076784.723654:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076784.723670:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076784.723674:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076784.723683:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3773 00000100:00080000:2.0:1769076784.723691:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076784.723699:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076784.723710:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076784.723741:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617307136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076784.723776:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769076784.723777:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076784.723789:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076784.723794:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076784.723806:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3773 00000100:00080000:2.0:1769076784.723815:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076784.723822:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076784.723834:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076784.723855:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617307264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076784.723874:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076784.723878:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076784/real 1769076784] req@ffff9b03482c4e00 x1855007617306880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076784.723886:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076784.723896:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076784.723927:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3773 00000100:00100000:1.0:1769076784.723932:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617306880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076784.723940:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076784.723958:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076784.723969:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769076784.723992:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1769076784.723996:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617307392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076784.724012:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076784.724046:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617306880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076784.724095:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076784.724172:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076784.724197:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076784/real 1769076784] req@ffff9b03482c5c00 x1855007617307008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076784.724222:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617307008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076784.724274:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076784.724281:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076784.724288:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076784.724300:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076784.724369:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076784.724393:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076784/real 1769076784] req@ffff9b03482c5f80 x1855007617307136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076784.724416:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617307136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076784.724437:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076784.724445:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076784.724450:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076784.724462:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076784.724516:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076784.724537:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076784/real 1769076784] req@ffff9b03482c7b80 x1855007617307264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076784.724557:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617307264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076784.724570:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076784.724576:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076784.724581:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076784.724591:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076784.724624:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076784.724642:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076784/real 1769076784] req@ffff9b03482c5880 x1855007617307392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076784.724663:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617307392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076784.724686:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076784.724693:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076784.724699:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076785.235541:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076785.619353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076785.619439:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076785.619446:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076786.259404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076786.643335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076786.643492:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076786.643546:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076787.283683:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076787.667386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076787.667387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076787.667491:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076788.307322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076788.691323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076788.691370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076788.691574:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076789.331304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076789.715354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076789.715382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076789.715473:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076789.843344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076789.843354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076789.843358:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076789.843367:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3779 00000100:00080000:1.0:1769076789.843376:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076789.843383:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076789.843414:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617307520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076789.843440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076789.843442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076789.843444:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076789.843447:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3779 00000100:00080000:1.0:1769076789.843449:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076789.843452:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076789.843460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076789.843469:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617307648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076789.843476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076789.843478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076789.843480:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076789.843483:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3779 00000100:00080000:1.0:1769076789.843484:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076789.843487:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076789.843491:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076789.843500:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617307520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076789.843501:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617307776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076789.843527:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076789.843529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076789.843531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076789.843533:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3779 00000100:00080000:1.0:1769076789.843536:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076789.843538:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076789.843542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076789.843553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617307904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076789.843567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076789.843569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076789.843570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076789.843573:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3779 00000100:00080000:1.0:1769076789.843577:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076789.843580:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076789.843583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076789.843590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617308032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076789.843611:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076789.843676:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076789/real 1769076789] req@ffff9b0343def100 x1855007617307520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076789.843695:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617307520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076789.843705:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076789.843710:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076789.843713:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617307520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076789.843720:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076789.843739:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076789.843748:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076789/real 1769076789] req@ffff9b0343dedf80 x1855007617307648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076789.843756:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617307648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076789.843763:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076789.843765:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076789.843767:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076789.843794:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076789.843805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076789.843812:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076789/real 1769076789] req@ffff9b0343ded880 x1855007617307776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076789.843820:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617307776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076789.843825:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076789.843827:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076789.843830:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076789.843837:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076789.843866:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076789.843879:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076789/real 1769076789] req@ffff9b0343def800 x1855007617307904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076789.843892:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617307904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076789.843900:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076789.843903:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076789.843905:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617307904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076789.843910:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076789.843933:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076789.843941:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076789/real 1769076789] req@ffff9b0343deea00 x1855007617308032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076789.843950:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617308032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076789.843956:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076789.843958:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076789.843960:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076790.355364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076790.739313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076790.739388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076790.739451:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076791.379465:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076791.763347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076791.763353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076791.763427:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076792.403338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076792.787382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076792.787451:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076792.787453:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076793.427369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076793.811328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076793.811366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076793.811458:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076794.451387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076794.835367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076794.835394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076794.835424:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076794.963366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076794.963382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076794.963389:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076794.963403:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3784 00000100:00080000:3.0:1769076794.963418:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076794.963427:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076794.963480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617308160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076794.963515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076794.963519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076794.963521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076794.963526:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3784 00000100:00080000:3.0:1769076794.963530:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076794.963534:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076794.963546:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076794.963561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617308288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076794.963575:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076794.963579:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076794.963581:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076794.963586:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3784 00000100:00080000:3.0:1769076794.963591:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076794.963601:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617308160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076794.963601:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076794.963608:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076794.963629:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617308416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076794.963647:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076794.963651:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076794.963653:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076794.963658:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3784 00000100:00080000:3.0:1769076794.963663:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076794.963668:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076794.963676:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076794.963689:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617308544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076794.963699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076794.963703:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076794.963705:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076794.963710:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3784 00000800:00000100:2.0:1769076794.963719:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076794.963720:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076794.963732:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076794.963738:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076794.963765:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617308672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076794.963802:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076794/real 1769076794] req@ffff9b0352a18700 x1855007617308160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076794.963841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617308160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076794.963859:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076794.963868:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076794.963874:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617308160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076794.963909:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076794.963950:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076794.963967:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076794/real 1769076794] req@ffff9b0352a1aa00 x1855007617308288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076794.963981:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617308288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076794.963990:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076794.963994:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076794.963998:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076794.964005:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076794.964023:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076794.964032:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076794/real 1769076794] req@ffff9b0352a18e00 x1855007617308416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076794.964044:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617308416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076794.964052:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076794.964055:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076794.964059:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076794.964064:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076794.964081:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076794.964090:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076794/real 1769076794] req@ffff9b0352a1a300 x1855007617308544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076794.964101:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617308544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076794.964109:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076794.964112:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076794.964114:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076794.964120:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076794.964134:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076794.964143:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076794/real 1769076794] req@ffff9b0352a19f80 x1855007617308672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076794.964153:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617308672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076794.964197:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076794.964201:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076794.964205:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076795.475361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076795.859353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076795.859353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076795.859426:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076796.499379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076796.883397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076796.883399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076796.883404:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076797.523407:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076797.907397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076797.907398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076797.907519:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076798.547334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076798.931352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076798.931353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076798.931459:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076799.571348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076799.955325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076799.955331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076799.955416:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076800.083407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076800.083417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076800.083422:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076800.083431:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3789 00000100:00080000:2.0:1769076800.083441:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076800.083446:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076800.083479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617308800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076800.083505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076800.083508:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076800.083509:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076800.083513:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3789 00000100:00080000:2.0:1769076800.083515:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076800.083518:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076800.083525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076800.083533:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617308928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076800.083541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076800.083544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076800.083545:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076800.083548:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3789 00000100:00080000:2.0:1769076800.083550:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076800.083553:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076800.083557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076800.083566:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617308800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076800.083567:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617309056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076800.083581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076800.083583:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076800.083584:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076800.083587:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3789 00000100:00080000:2.0:1769076800.083598:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076800.083601:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076800.083605:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076800.083613:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617309184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076800.083619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076800.083622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076800.083623:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076800.083626:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3789 00000100:00080000:2.0:1769076800.083628:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076800.083630:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076800.083633:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076800.083642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617309312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076800.083650:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076800.083694:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076800/real 1769076800] req@ffff9b03482c6680 x1855007617308800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076800.083711:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617308800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076800.083721:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076800.083725:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076800.083728:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617308800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076800.083735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076800.083753:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076800.083764:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076800/real 1769076800] req@ffff9b03482c6a00 x1855007617308928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076800.083775:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617308928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076800.083783:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076800.083786:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076800.083789:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617308928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076800.083809:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076800.083827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076800.083836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076800/real 1769076800] req@ffff9b03482c4000 x1855007617309056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076800.083847:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617309056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076800.083854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076800.083857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076800.083860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076800.083865:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076800.083880:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076800.083888:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076800/real 1769076800] req@ffff9b03482c7100 x1855007617309184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076800.083898:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617309184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076800.083906:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076800.083908:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076800.083911:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076800.083916:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076800.083929:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076800.083937:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076800/real 1769076800] req@ffff9b03482c4700 x1855007617309312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076800.083954:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617309312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076800.083961:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076800.083963:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076800.083967:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769076800.595404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076800.979363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076800.979393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076800.979467:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076801.619400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076802.003333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076802.003354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076802.003386:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076802.643426:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076803.027386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076803.027391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076803.027566:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076803.667330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076804.051318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076804.051322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076804.051427:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076804.691404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076805.075345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076805.075366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076805.075437:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076805.203315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076805.203342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076805.203347:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076805.203357:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3794 00000100:00080000:2.0:1769076805.203369:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076805.203377:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076805.203433:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617309440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076805.203462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076805.203466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076805.203468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076805.203472:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3794 00000100:00080000:2.0:1769076805.203476:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076805.203479:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076805.203489:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076805.203501:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617309568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076805.203514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076805.203517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076805.203519:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076805.203528:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3794 00000100:00100000:1.0:1769076805.203531:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617309440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076805.203532:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076805.203546:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076805.203552:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076805.203571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617309696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076805.203579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076805.203592:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076805.203594:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076805.203599:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3794 00000100:00080000:2.0:1769076805.203603:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076805.203606:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076805.203611:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076805.203645:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076805.203646:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617309824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076805.203660:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076805.203665:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076805.203667:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076805.203674:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3794 00000100:00080000:2.0:1769076805.203695:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076805.203700:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076805.203707:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076805.203723:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076805/real 1769076805] req@ffff9b03482c6d80 x1855007617309440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076805.203724:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617309952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076805.203767:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617309440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076805.203778:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076805.203782:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076805.203786:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617309440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076805.203813:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076805.203833:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076805.203842:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076805/real 1769076805] req@ffff9b03482c6300 x1855007617309568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076805.203851:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617309568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076805.203858:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076805.203860:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076805.203863:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076805.203867:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076805.203878:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076805.203884:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076805/real 1769076805] req@ffff9b03482c4a80 x1855007617309696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076805.203891:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617309696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076805.203897:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076805.203899:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076805.203901:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076805.203905:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076805.203915:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076805.203920:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076805/real 1769076805] req@ffff9b03482c5180 x1855007617309824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076805.203933:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617309824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076805.203938:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076805.203940:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076805.203941:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076805.203945:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076805.203955:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076805.203960:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076805/real 1769076805] req@ffff9b03482c4380 x1855007617309952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076805.203966:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617309952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076805.203972:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076805.203975:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076805.203977:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617309952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076805.715340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076806.099305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076806.099347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076806.099390:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076806.739352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076807.123367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076807.123367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076807.123458:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076807.763420:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076808.147313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076808.147348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076808.147426:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076808.787329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076809.171318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076809.171318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076809.171566:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076809.811413:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076810.195343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076810.195343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076810.195458:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076810.323432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076810.323448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076810.323454:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076810.323468:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3799 00000100:00080000:1.0:1769076810.323482:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076810.323493:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076810.323537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617310080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076810.323569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076810.323574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076810.323576:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076810.323580:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3799 00000100:00080000:1.0:1769076810.323584:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076810.323589:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076810.323600:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076810.323612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617310208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076810.323625:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617310080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076810.323628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076810.323642:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076810.323644:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076810.323650:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3799 00000100:00080000:1.0:1769076810.323654:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076810.323658:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076810.323665:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076810.323676:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617310336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076810.323686:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076810.323690:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076810.323691:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076810.323695:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3799 00000100:00080000:1.0:1769076810.323706:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076810.323709:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076810.323715:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769076810.323722:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076810.323730:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617310464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076810.323742:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076810.323746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076810.323748:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076810.323752:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3799 00000100:00080000:1.0:1769076810.323755:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076810.323759:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076810.323763:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076810.323791:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617310592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076810.323796:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076810/real 1769076810] req@ffff9b0343deed80 x1855007617310080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076810.323824:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617310080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076810.323847:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076810.323852:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076810.323857:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617310080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076810.323882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076810.323930:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076810.323956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076810/real 1769076810] req@ffff9b0343deca80 x1855007617310208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076810.323972:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617310208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076810.323982:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076810.323986:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076810.323990:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076810.324020:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076810.324052:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076810.324069:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076810/real 1769076810] req@ffff9b0343dece00 x1855007617310336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076810.324102:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617310336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076810.324116:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076810.324122:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076810.324128:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076810.324138:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076810.324180:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076810.324199:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076810/real 1769076810] req@ffff9b0343defb80 x1855007617310464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076810.324219:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617310464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076810.324232:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076810.324237:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076810.324242:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076810.324290:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076810.324310:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076810.324320:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076810/real 1769076810] req@ffff9b0343dee680 x1855007617310592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076810.324333:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617310592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076810.324341:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076810.324344:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076810.324348:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076810.835336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076811.219297:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076811.219312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076811.219361:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076811.859402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076812.243372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076812.243372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076812.243442:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076812.883396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076813.267333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076813.267333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076813.267415:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076813.907308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076814.291339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076814.291340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076814.291422:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076814.931365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076815.315352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076815.315409:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076815.315485:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076815.443388:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076815.443400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076815.443405:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076815.443416:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3804 00000100:00080000:3.0:1769076815.443427:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076815.443436:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076815.443479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617310720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076815.443510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076815.443514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076815.443515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076815.443518:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3804 00000100:00080000:3.0:1769076815.443528:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076815.443532:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076815.443544:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076815.443559:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617310848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076815.443567:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617310720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076815.443595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076815.443600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076815.443603:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076815.443612:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3804 00000100:00080000:3.0:1769076815.443617:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076815.443620:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076815.443625:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076815.443636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617310976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076815.443643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076815.443646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076815.443647:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076815.443650:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3804 00000100:00080000:3.0:1769076815.443653:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076815.443660:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076815.443664:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076815.443669:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076815.443681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617311104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076815.443688:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076815.443691:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076815.443693:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769076815.443704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076815/real 1769076815] req@ffff9b0352a19f80 x1855007617310720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076815.443720:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617310720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076815.443729:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076815.443734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076815.443737:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617310720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076815.443758:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310848:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769076815.443771:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3804 00000100:00080000:3.0:1769076815.443776:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769076815.443784:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076815.443787:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076815.443794:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076815.443805:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617311232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076815.443810:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076815/real 1769076815] req@ffff9b0352a1a300 x1855007617310848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076815.443855:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617310848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076815.443864:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076815.443869:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076815.443872:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076815.443878:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076815.443896:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076815.443905:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076815/real 1769076815] req@ffff9b0352a18e00 x1855007617310976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076815.443914:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617310976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076815.443920:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076815.443922:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076815.443924:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617310976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076815.443928:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076815.443942:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076815.443958:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076815/real 1769076815] req@ffff9b0352a1aa00 x1855007617311104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076815.443966:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617311104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076815.443972:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076815.443974:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076815.443977:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076815.443994:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076815.444009:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076815.444022:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076815/real 1769076815] req@ffff9b0352a18700 x1855007617311232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076815.444032:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617311232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076815.444039:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076815.444041:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076815.444048:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076815.955384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076816.339328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076816.339328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076816.339483:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076816.979325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076817.363336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076817.363342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076817.363439:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076818.003326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076818.387405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076818.387414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076818.387519:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076819.027382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076819.411395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076819.411415:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076819.411608:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076820.051460:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076820.435396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076820.435396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076820.435534:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076820.563369:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076820.563385:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076820.563392:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076820.563408:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3809 00000100:00080000:1.0:1769076820.563424:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076820.563435:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076820.563505:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617311360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076820.563544:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076820.563549:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076820.563558:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076820.563569:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3809 00000100:00080000:1.0:1769076820.563573:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076820.563579:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076820.563591:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076820.563606:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617311488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076820.563619:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076820.563624:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769076820.563626:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617311360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076820.563627:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076820.563656:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3809 00000100:00080000:1.0:1769076820.563663:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076820.563667:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076820.563673:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076820.563691:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617311616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076820.563702:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076820.563706:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076820.563709:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076820.563712:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3809 00000100:00080000:1.0:1769076820.563715:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076820.563719:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076820.563726:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076820.563785:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617311744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076820.563795:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076820.563801:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076820.563804:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076820.563810:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3809 00000100:00080000:1.0:1769076820.563816:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076820.563823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076820.563828:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076820.563838:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076820.563856:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617311872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076820.563895:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076820/real 1769076820] req@ffff9b0343def480 x1855007617311360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076820.563917:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617311360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076820.563928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076820.563933:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076820.563937:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617311360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076820.563964:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076820.563989:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076820.564000:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076820/real 1769076820] req@ffff9b0343ded500 x1855007617311488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076820.564011:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617311488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076820.564020:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076820.564023:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076820.564026:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076820.564031:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076820.564056:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076820.564065:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076820/real 1769076820] req@ffff9b0343dee680 x1855007617311616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076820.564076:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617311616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076820.564083:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076820.564086:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076820.564089:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076820.564094:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076820.564108:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076820.564115:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076820/real 1769076820] req@ffff9b0343defb80 x1855007617311744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076820.564126:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617311744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076820.564133:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076820.564135:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076820.564138:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076820.564143:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076820.564155:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076820.564162:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076820/real 1769076820] req@ffff9b0343dece00 x1855007617311872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076820.564171:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617311872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076820.564178:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076820.564180:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076820.564183:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617311872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076821.075383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076821.459329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076821.459407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076821.459511:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076822.099370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076822.483417:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076822.483500:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076822.483665:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076823.123441:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076823.507392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076823.507538:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076823.507577:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076824.147351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076824.531362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076824.531410:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076824.531476:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076825.171409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076825.555382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076825.555486:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076825.555509:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076825.683501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076825.683514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076825.683521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076825.683553:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3814 00000100:00080000:1.0:1769076825.683567:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076825.683577:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076825.683629:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617312000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076825.683688:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076825.683695:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076825.683698:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076825.683707:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3814 00000100:00080000:1.0:1769076825.683713:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076825.683719:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076825.683732:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076825.683755:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617312000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076825.683756:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617312128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076825.683815:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076825.683819:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076825.683823:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076825.683834:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3814 00000100:00080000:1.0:1769076825.683840:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076825.683845:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076825.683852:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076825.683870:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617312256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076825.683885:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076825.683889:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076825.683891:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076825.683897:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3814 00000100:00080000:1.0:1769076825.683932:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076825.683942:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076825.683949:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076825.683963:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076825.683985:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617312384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076825.683998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076825.684002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076825.684005:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076825.684013:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3814 00000100:00080000:1.0:1769076825.684020:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076825.684035:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076825.684036:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076825/real 1769076825] req@ffff9b0343deca80 x1855007617312000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076825.684053:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076825.684080:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617312512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076825.684095:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617312000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076825.684114:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076825.684126:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076825.684134:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617312000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076825.684176:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076825.684207:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076825.684223:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076825/real 1769076825] req@ffff9b0343deed80 x1855007617312128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076825.684331:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617312128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076825.684352:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076825.684359:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076825.684366:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076825.684380:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076825.684415:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076825.684438:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076825/real 1769076825] req@ffff9b0343deea00 x1855007617312256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076825.684463:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617312256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076825.684474:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076825.684478:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076825.684484:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076825.684495:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076825.684534:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076825.684557:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076825/real 1769076825] req@ffff9b0343def800 x1855007617312384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076825.684583:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617312384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076825.684601:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076825.684608:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076825.684615:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076825.684628:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076825.684698:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076825.684715:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076825/real 1769076825] req@ffff9b0343ded880 x1855007617312512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076825.684730:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617312512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076825.684759:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076825.684764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076825.684768:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076826.195354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076826.579358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076826.579372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076826.579451:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076827.219383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076827.603363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076827.603675:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076827.603889:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076828.243435:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076828.627423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076828.627760:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076828.627763:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076829.267358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076829.651511:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076829.651526:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076829.651631:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076830.291364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076830.675350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076830.675356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076830.675510:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076830.803347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076830.803356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076830.803360:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076830.803369:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3819 00000100:00080000:3.0:1769076830.803377:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076830.803382:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076830.803415:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617312640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076830.803439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076830.803443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076830.803445:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076830.803450:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3819 00000100:00080000:3.0:1769076830.803454:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076830.803458:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076830.803467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076830.803480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617312768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076830.803485:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617312640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076830.803491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076830.803494:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076830.803496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076830.803500:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3819 00000100:00080000:3.0:1769076830.803504:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076830.803508:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076830.803512:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076830.803520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617312896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076830.803528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076830.803531:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076830.803533:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076830.803536:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3819 00000100:00080000:3.0:1769076830.803539:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076830.803543:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769076830.803547:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769076830.803548:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076830.803556:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617313024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076830.803605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076830.803608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076830.803610:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076830.803614:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3819 00000100:00100000:2.0:1769076830.803617:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076830/real 1769076830] req@ffff9b0352a1a680 x1855007617312640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076830.803617:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076830.803621:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076830.803626:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076830.803636:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617312640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076830.803637:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617313152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076830.803648:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076830.803652:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076830.803657:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617312640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076830.803679:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076830.803702:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076830.803713:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076830/real 1769076830] req@ffff9b0352a18a80 x1855007617312768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076830.803726:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617312768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076830.803732:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076830.803734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076830.803736:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076830.803740:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076830.803752:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076830.803758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076830/real 1769076830] req@ffff9b0352a1b480 x1855007617312896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076830.803766:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617312896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076830.803770:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076830.803772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076830.803774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617312896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076830.803777:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076830.803787:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076830.803793:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076830/real 1769076830] req@ffff9b0352a1ad80 x1855007617313024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076830.803800:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617313024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076830.803809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076830.803811:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076830.803813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076830.803816:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076830.803825:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076830.803830:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076830/real 1769076830] req@ffff9b0352a1b100 x1855007617313152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076830.803836:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617313152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076830.803841:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076830.803843:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076830.803845:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076831.315342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076831.699337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076831.699354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076831.699365:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076832.339392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076832.723341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076832.723348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076832.723514:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076833.363406:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076833.747358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076833.747359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076833.747396:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076834.387330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076834.771402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076834.771404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076834.771563:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076835.411392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076835.795390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076835.795539:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076835.795547:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769076835.923392:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076835.923410:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076835.923417:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076835.923434:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3825 00000100:00080000:3.0:1769076835.923451:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076835.923466:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076835.923574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617313280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076835.923625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076835.923631:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076835.923634:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076835.923640:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3825 00000100:00080000:3.0:1769076835.923647:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076835.923654:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076835.923672:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076835.923695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617313280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076835.923698:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617313408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076835.923737:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076835.923743:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076835.923745:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076835.923751:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3825 00000100:00080000:3.0:1769076835.923758:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076835.923762:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076835.923770:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076835.923793:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617313536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076835.923828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076835.923834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076835.923838:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076835.923846:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3825 00000100:00080000:3.0:1769076835.923854:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769076835.923858:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076835.923868:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076835.923881:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076835.923915:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617313664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076835.923927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076835.923931:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769076835.923936:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076835/real 1769076835] req@ffff9b0352a18380 x1855007617313280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076835.923937:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076835.923951:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3825 00000100:00080000:3.0:1769076835.923961:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076835.923969:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076835.923979:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076835.923999:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617313792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076835.924020:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617313280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076835.924037:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076835.924045:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076835.924053:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617313280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076835.924088:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076835.924116:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076835.924131:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076835/real 1769076835] req@ffff9b0352a1b800 x1855007617313408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076835.924145:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617313408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076835.924155:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076835.924159:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076835.924162:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076835.924170:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076835.924218:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076835.924234:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076835/real 1769076835] req@ffff9b0352a19500 x1855007617313536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076835.924289:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617313536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076835.924321:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076835.924329:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076835.924336:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076835.924349:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076835.924394:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076835.924420:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076835/real 1769076835] req@ffff9b0352a18000 x1855007617313664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076835.924453:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617313664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076835.924518:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076835.924522:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076835.924526:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076835.924534:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076835.924578:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076835.924595:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076835/real 1769076835] req@ffff9b0352a19c00 x1855007617313792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076835.924610:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617313792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076835.924620:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076835.924624:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076835.924628:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617313792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076836.435402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076836.819397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076836.819400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076836.819509:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076837.459331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076837.843334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076837.843334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076837.843415:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076838.483501:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076838.867349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076838.867371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076838.867510:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076839.507397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076839.891360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076839.891480:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076839.891576:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076840.531450:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076840.915325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076840.915328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076840.915402:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076841.043415:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076841.043425:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076841.043428:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076841.043437:0:59411:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3830 00000100:00080000:3.0:1769076841.043446:0:59411:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076841.043454:0:59411:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076841.043489:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617313920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076841.043511:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076841.043514:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076841.043515:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076841.043518:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3830 00000100:00080000:3.0:1769076841.043520:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076841.043523:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076841.043529:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076841.043537:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617314048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076841.043545:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076841.043548:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076841.043550:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076841.043552:0:59411:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3830 00000100:00080000:3.0:1769076841.043554:0:59411:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076841.043557:0:59411:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076841.043560:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076841.043570:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617314176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076841.043571:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617313920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076841.043584:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076841.043587:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076841.043589:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076841.043592:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3830 00000100:00080000:3.0:1769076841.043593:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076841.043596:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076841.043599:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076841.043607:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617314304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076841.043613:0:59411:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076841.043615:0:59411:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076841.043616:0:59411:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076841.043619:0:59411:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3830 00000100:00080000:3.0:1769076841.043631:0:59411:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076841.043634:0:59411:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076841.043637:0:59411:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769076841.043646:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076841.043647:0:59411:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617314432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076841.043692:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9b0352a19c00 x1855007617313920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076841.043722:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617313920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076841.043734:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.043738:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076841.043742:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617313920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076841.043751:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076841.043776:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076841.043788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9b0352a18000 x1855007617314048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076841.043801:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617314048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076841.043810:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.043814:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076841.043818:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076841.043841:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076841.043860:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076841.043870:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9b0352a19500 x1855007617314176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076841.043883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617314176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076841.043892:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.043896:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076841.043900:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076841.043906:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076841.043933:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076841.043945:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9b0352a1b800 x1855007617314304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076841.043958:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617314304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076841.043967:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.043970:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076841.043974:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076841.043981:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076841.044000:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076841.044020:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076841/real 1769076841] req@ffff9b0352a18380 x1855007617314432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076841.044033:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617314432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076841.044048:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076841.044052:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076841.044055:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076841.555331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076841.939353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076841.939412:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076841.939508:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076842.579415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076842.963326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076842.963450:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076842.963551:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076843.603535:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076843.987376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076843.987406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076843.987483:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076844.627324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076845.011314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076845.011332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076845.011430:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076845.651357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076846.035289:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076846.035346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076846.035434:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076846.163409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076846.163424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076846.163428:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076846.163438:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3835 00000100:00080000:2.0:1769076846.163448:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076846.163456:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076846.163493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617314560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076846.163524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076846.163530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076846.163533:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076846.163539:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3835 00000100:00080000:2.0:1769076846.163543:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076846.163548:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076846.163560:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076846.163573:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617314560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076846.163578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617314688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076846.163591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076846.163595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076846.163597:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076846.163602:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3835 00000100:00080000:2.0:1769076846.163606:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076846.163611:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076846.163617:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076846.163628:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617314816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076846.163640:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076846.163644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076846.163646:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076846.163651:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3835 00000100:00080000:2.0:1769076846.163656:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076846.163660:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076846.163666:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076846.163678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617314944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076846.163690:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076846.163694:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314688:192.168.204.160@tcp:38:kworker.0 00000100:00080000:2.0:1769076846.163694:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076846.163697:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076846.163704:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3835 00000100:00080000:2.0:1769076846.163709:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076846.163714:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076846.163720:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076846.163731:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076846.163736:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617315072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076846.163759:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076846.163793:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076846.675302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076847.059360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076847.059378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076847.059486:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076847.699316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076848.083351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076848.083363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076848.083455:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076848.723318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076849.107325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076849.107342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076849.107375:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769076849.235843:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:3.0:1769076849.235947:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769076849.235976:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769076849.236057:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769076849.236082:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769076849.236106:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076849.236108:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076846/real 1769076849] req@ffff9b03482c4380 x1855007617314688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1769076849.236129:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076849.236152:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617314688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076849.236165:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076849.236171:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076849.236174:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076849.236181:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076846/real 1769076849] req@ffff9b03482c5180 x1855007617314816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076849.236191:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617314816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076849.236197:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076849.236199:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076849.236202:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076849.236206:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076846/real 1769076849] req@ffff9b03482c4a80 x1855007617314944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076849.236214:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617314944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076849.236223:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076849.236227:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076849.236229:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617314944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076849.236262:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076846/real 1769076849] req@ffff9b03482c5500 x1855007617314560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076849.236281:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617314560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076849.236293:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076849.236298:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076849.236302:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617314560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076849.236345:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076846/real 1769076849] req@ffff9b03482c6300 x1855007617315072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076849.236360:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617315072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076849.236368:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076849.236372:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076849.236376:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076849.747376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076850.131373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076850.131497:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076850.131503:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076850.771404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076851.155345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076851.155345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076851.155467:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076851.283486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076851.283503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076851.283511:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076851.283529:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3840 00000100:00080000:1.0:1769076851.283548:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076851.283561:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076851.283631:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617315200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076851.283711:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076851.283720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076851.283725:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076851.283737:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3840 00000100:00080000:1.0:1769076851.283749:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076851.283761:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076851.283782:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076851.283818:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617315328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076851.283829:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617315200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076851.283857:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076851.283868:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076851.283873:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076851.283887:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3840 00000100:00080000:1.0:1769076851.283898:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076851.283910:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076851.283922:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076851.283956:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076851.283958:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617315456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076851.283995:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076851.284002:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076851.284006:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076851.284018:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3840 00000100:00080000:1.0:1769076851.284027:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076851.284036:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076851.284048:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076851.284053:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076851/real 1769076851] req@ffff9b0343dedf80 x1855007617315200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076851.284079:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617315584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076851.284093:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617315200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076851.284119:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076851.284126:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076851.284130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076851.284141:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076851.284145:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769076851.284161:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3840 00000100:00100000:2.0:1769076851.284165:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617315200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076851.284178:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076851.284194:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076851.284204:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076851.284298:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076851.284311:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617315712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076851.284361:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076851.284393:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076851/real 1769076851] req@ffff9b0343def100 x1855007617315328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076851.284417:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617315328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076851.284432:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076851.284439:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076851.284445:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076851.284458:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076851.284494:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076851.284513:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076851/real 1769076851] req@ffff9b0343dee300 x1855007617315456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076851.284532:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617315456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076851.284543:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076851.284548:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076851.284553:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076851.284563:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076851.284594:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076851.284611:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076851/real 1769076851] req@ffff9b0343dec380 x1855007617315584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076851.284629:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617315584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076851.284650:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076851.284655:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076851.284660:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076851.284687:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076851.284715:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076851.284731:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076851/real 1769076851] req@ffff9b0343ded180 x1855007617315712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076851.284762:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617315712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076851.284789:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076851.284795:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076851.284799:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076851.795356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076852.179341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076852.179414:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076852.179480:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076852.819368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076853.203308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076853.203376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076853.203467:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076853.843367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076854.227371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076854.227383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076854.227474:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076854.867305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076855.251421:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076855.251421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076855.251557:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076855.891310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076856.275367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076856.275367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076856.275465:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076856.403382:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076856.403390:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076856.403394:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076856.403402:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3845 00000100:00080000:3.0:1769076856.403410:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076856.403416:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076856.403455:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617315840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076856.403483:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076856.403486:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076856.403487:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076856.403491:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3845 00000100:00080000:3.0:1769076856.403494:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076856.403496:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076856.403504:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076856.403514:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617315968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076856.403522:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076856.403524:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076856.403526:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076856.403530:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3845 00000100:00100000:2.0:1769076856.403532:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617315840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076856.403533:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076856.403554:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076856.403560:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076856.403568:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617316096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076856.403574:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076856.403576:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076856.403578:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076856.403581:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3845 00000100:00080000:3.0:1769076856.403584:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076856.403586:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076856.403590:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076856.403599:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617316224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076856.403606:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076856.403608:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076856.403609:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076856.403612:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3845 00000100:00080000:3.0:1769076856.403615:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076856.403617:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076856.403620:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769076856.403625:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076856.403630:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617316352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076856.403663:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076856/real 1769076856] req@ffff9b0352a1b100 x1855007617315840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076856.403682:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617315840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076856.403690:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076856.403694:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076856.403697:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617315840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076856.403715:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076856.403730:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076856.403738:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076856/real 1769076856] req@ffff9b0352a1ad80 x1855007617315968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076856.403746:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617315968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076856.403752:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076856.403754:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076856.403756:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617315968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076856.403760:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076856.403770:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076856.403775:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076856/real 1769076856] req@ffff9b0352a1b480 x1855007617316096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076856.403794:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617316096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076856.403805:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076856.403807:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076856.403809:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076856.403813:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076856.403841:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076856.403847:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076856/real 1769076856] req@ffff9b0352a18a80 x1855007617316224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076856.403855:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617316224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076856.403860:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076856.403861:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076856.403863:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076856.403867:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076856.403877:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076856.403882:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076856/real 1769076856] req@ffff9b0352a1a680 x1855007617316352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076856.403890:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617316352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076856.403913:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076856.403914:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076856.403917:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076856.915346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076857.299355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076857.299416:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076857.299436:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076857.939334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076858.323382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076858.323382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076858.323496:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076858.963343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076859.347367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076859.347407:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076859.347436:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076859.987336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076860.371438:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076860.371450:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076860.371550:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076861.011365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076861.395350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076861.395413:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076861.395525:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076861.523531:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076861.523551:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076861.523556:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076861.523565:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3850 00000100:00080000:0.0:1769076861.523594:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076861.523602:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076861.523643:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617316480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076861.523710:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076861.523716:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076861.523719:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076861.523725:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3850 00000100:00080000:0.0:1769076861.523731:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076861.523740:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076861.523752:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076861.523772:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617316480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076861.523779:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617316608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076861.523798:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076861.523802:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076861.523804:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076861.523808:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3850 00000100:00080000:0.0:1769076861.523813:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076861.523817:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076861.523823:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076861.523835:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617316736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076861.523845:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076861.523849:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076861.523850:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076861.523855:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3850 00000800:00000100:2.0:1769076861.523862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076861.523864:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076861.523871:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076861.523877:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076861.523901:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617316864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076861.523910:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076861.523913:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076861.523916:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076861.523920:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3850 00000100:00080000:0.0:1769076861.523948:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076861.523950:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076861/real 1769076861] req@ffff9b0365a3bb80 x1855007617316480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076861.523958:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076861.523971:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076861.523988:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617316480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076861.523993:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617316992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076861.524025:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076861.524036:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076861.524062:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617316480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076861.524111:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076861.524137:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076861.524156:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076861/real 1769076861] req@ffff9b0365a3b100 x1855007617316608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076861.524175:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617316608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076861.524184:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076861.524187:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076861.524191:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076861.524197:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076861.524212:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076861.524220:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076861/real 1769076861] req@ffff9b0365a39500 x1855007617316736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076861.524230:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617316736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076861.524308:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076861.524312:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076861.524316:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076861.524323:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076861.524348:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076861.524360:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076861/real 1769076861] req@ffff9b0365a39180 x1855007617316864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076861.524373:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617316864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076861.524388:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076861.524392:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076861.524395:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076861.524402:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076861.524417:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076861.524443:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076861/real 1769076861] req@ffff9b0365a3ad80 x1855007617316992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076861.524454:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617316992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076861.524462:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076861.524465:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076861.524467:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617316992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076862.035315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076862.419372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076862.419412:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076862.419536:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076863.059357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076863.443365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076863.443376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076863.443549:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076864.083390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076864.467313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076864.467383:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076864.467406:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076865.107380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076865.491316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076865.491349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076865.491452:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076866.131371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076866.515368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076866.515382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076866.515514:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076866.643443:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076866.643460:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076866.643468:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076866.643485:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3855 00000100:00080000:0.0:1769076866.643503:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076866.643515:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076866.643585:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617317120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076866.643653:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076866.643661:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076866.643665:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076866.643674:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3855 00000100:00080000:0.0:1769076866.643681:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076866.643689:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076866.643708:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076866.643741:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617317248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076866.643766:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076866.643773:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076866.643779:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076866.643781:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617317120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076866.643789:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3855 00000100:00080000:0.0:1769076866.643804:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076866.643810:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076866.643818:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076866.643841:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617317376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076866.643862:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076866.643869:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076866.643873:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076866.643881:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3855 00000100:00080000:0.0:1769076866.643891:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076866.643900:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769076866.643909:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769076866.643917:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076866.643951:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617317504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076866.643992:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076866.643997:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076866.644000:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076866.644007:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3855 00000100:00080000:0.0:1769076866.644015:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076866.644020:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076866/real 1769076866] req@ffff9b0365a3ad80 x1855007617317120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076866.644026:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076866.644040:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076866.644066:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617317120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076866.644069:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617317632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076866.644116:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076866.644128:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076866.644135:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617317120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076866.644174:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617317248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076866.644221:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076866.644287:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076866/real 1769076866] req@ffff9b0365a39180 x1855007617317248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076866.644317:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617317248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076866.644335:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076866.644342:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076866.644349:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617317248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076866.644363:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617317376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076866.644405:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076866.644425:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076866/real 1769076866] req@ffff9b0365a39500 x1855007617317376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076866.644451:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617317376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076866.644462:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076866.644465:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076866.644470:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617317376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076866.644478:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617317504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076866.644502:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076866.644514:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076866/real 1769076866] req@ffff9b0365a3b100 x1855007617317504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076866.644529:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617317504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076866.644539:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076866.644542:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076866.644546:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617317504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076866.644553:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617317632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076866.644571:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076866.644581:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076866/real 1769076866] req@ffff9b0365a3bb80 x1855007617317632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076866.644595:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617317632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076866.644613:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076866.644617:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076866.644621:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617317632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076867.155440:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076867.539382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076867.539382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076867.539608:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076868.179377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076868.563376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076868.563378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076868.563439:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076869.203397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076869.587374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076869.587374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076869.587430:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076870.227311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076870.611356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076870.611376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076870.611517:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076871.251383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076871.635345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076871.635533:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076871.635567:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:1.0:1769076871.763406:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076871.763424:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076871.763431:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076871.763448:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3860 00000100:00080000:1.0:1769076871.763464:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076871.763475:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076871.763529:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617317760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076871.763571:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076871.763577:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076871.763580:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076871.763585:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3860 00000100:00080000:1.0:1769076871.763589:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076871.763594:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076871.763606:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076871.763621:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617317888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076871.763643:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076871.763648:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076871.763651:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076871.763658:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3860 00000100:00080000:1.0:1769076871.763662:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076871.763667:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076871.763672:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076871.763688:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617318016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076871.763698:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617317760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076871.763706:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076871.763725:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076871.763728:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076871.763734:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3860 00000100:00080000:1.0:1769076871.763737:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076871.763742:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076871.763749:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076871.763765:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617318144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076871.763777:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076871.763782:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076871.763785:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076871.763789:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3860 00000100:00080000:1.0:1769076871.763796:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076871.763801:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076871.763809:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076871.763825:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617318272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076871.763832:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076871.763947:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076871/real 1769076871] req@ffff9b0343dec700 x1855007617317760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076871.763986:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617317760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076871.764008:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076871.764015:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076871.764021:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617317760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076871.764032:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617317888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076871.764074:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076871.764092:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076871/real 1769076871] req@ffff9b0343dec000 x1855007617317888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076871.764108:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617317888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076871.764131:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076871.764135:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076871.764139:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617317888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076871.764174:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076871.764195:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076871.764207:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076871/real 1769076871] req@ffff9b0343ded180 x1855007617318016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076871.764221:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617318016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076871.764277:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076871.764282:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076871.764286:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076871.764295:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076871.764331:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076871.764345:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076871/real 1769076871] req@ffff9b0343dec380 x1855007617318144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076871.764360:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617318144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076871.764372:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076871.764376:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076871.764380:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076871.764387:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076871.764407:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076871.764417:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076871/real 1769076871] req@ffff9b0343dee300 x1855007617318272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076871.764430:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617318272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076871.764440:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076871.764443:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076871.764447:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076872.275345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076872.659415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076872.659415:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076872.659549:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076873.299461:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076873.683362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076873.683362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076873.683457:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076874.323400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076874.707424:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076874.707495:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076874.707562:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076875.347332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076875.731369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076875.731369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076875.731452:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076876.371381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076876.755348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076876.755377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076876.755522:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076876.883412:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076876.883426:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076876.883434:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076876.883450:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3866 00000100:00080000:0.0:1769076876.883465:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076876.883475:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076876.883548:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617318400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076876.883609:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076876.883618:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076876.883622:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076876.883631:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3866 00000100:00080000:0.0:1769076876.883639:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076876.883648:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076876.883669:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076876.883691:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617318400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076876.883702:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617318528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076876.883721:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076876.883726:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076876.883729:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076876.883734:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3866 00000100:00080000:0.0:1769076876.883740:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076876.883746:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076876.883754:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076876.883773:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617318656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076876.883857:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076876.883864:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076876.883871:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076876.883879:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076876.883891:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3866 00000100:00080000:0.0:1769076876.883903:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076876.883912:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076876.883927:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076876.883939:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076876/real 1769076876] req@ffff9b0365a3aa00 x1855007617318400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076876.883955:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617318784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076876.884015:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076876.884020:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076876.884023:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076876.884030:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3866 00000100:00100000:2.0:1769076876.884036:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617318400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076876.884040:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076876.884056:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076876.884080:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769076876.884095:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769076876.884107:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617318912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076876.884112:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076876.884125:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617318400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076876.884189:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076876.884290:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076876.884315:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076876/real 1769076876] req@ffff9b0365a3b480 x1855007617318528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076876.884340:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617318528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076876.884358:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076876.884365:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076876.884373:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076876.884386:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076876.884432:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076876.884455:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076876/real 1769076876] req@ffff9b0365a38e00 x1855007617318656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076876.884482:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617318656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076876.884524:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076876.884531:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076876.884539:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076876.884551:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076876.884607:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076876.884630:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076876/real 1769076876] req@ffff9b0365a39c00 x1855007617318784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076876.884655:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617318784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076876.884674:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076876.884681:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076876.884688:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076876.884700:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076876.884731:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076876.884745:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076876/real 1769076876] req@ffff9b0365a38a80 x1855007617318912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076876.884760:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617318912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076876.884771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076876.884776:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076876.884783:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617318912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076877.395397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076877.779322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076877.779322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076877.779400:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076878.419403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076878.803333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076878.803434:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076878.803443:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076879.443370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076879.827372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076879.827445:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076879.827636:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076880.467415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076880.851415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076880.851421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076880.851620:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076881.491401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076881.875343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076881.875343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076881.875542:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076882.003534:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076882.003547:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076882.003553:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076882.003565:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3871 00000100:00080000:2.0:1769076882.003580:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076882.003588:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076882.003639:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617319040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076882.003685:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076882.003691:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076882.003693:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076882.003730:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3871 00000100:00080000:2.0:1769076882.003735:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076882.003740:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076882.003762:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076882.003777:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617319168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076882.003792:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076882.003797:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076882.003801:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076882.003808:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3871 00000100:00080000:2.0:1769076882.003812:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076882.003818:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076882.003823:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076882.003835:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617319296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076882.003848:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617319040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076882.003853:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076882.003863:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076882.003869:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076882.003875:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3871 00000100:00080000:2.0:1769076882.003879:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076882.003884:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076882.003890:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076882.003907:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617319424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076882.003919:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076882.003924:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076882.003926:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076882.003931:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3871 00000100:00080000:2.0:1769076882.003935:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076882.003939:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076882.003945:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076882.003963:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617319552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076882.003967:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076882.004034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff9b03482c6d80 x1855007617319040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076882.004059:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617319040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076882.004073:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076882.004079:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076882.004085:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617319040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076882.004095:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076882.004119:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076882.004133:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff9b03482c4700 x1855007617319168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076882.004149:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617319168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076882.004159:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076882.004163:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076882.004167:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076882.004196:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076882.004215:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076882.004226:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff9b03482c7100 x1855007617319296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076882.004289:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617319296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076882.004300:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076882.004304:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076882.004308:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076882.004316:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076882.004340:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076882.004353:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff9b03482c4000 x1855007617319424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076882.004376:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617319424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076882.004386:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076882.004389:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076882.004392:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076882.004399:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076882.004416:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076882.004426:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076882/real 1769076882] req@ffff9b03482c6a00 x1855007617319552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076882.004447:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617319552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076882.004457:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076882.004460:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076882.004464:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076882.515370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076882.899388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076882.899413:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076882.899490:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076883.539389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076883.923366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076883.923464:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076883.923557:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076884.563420:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076884.947388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076884.947424:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076884.947524:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076885.587367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076885.971361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076885.971447:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076885.971517:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076886.611309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076886.995372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076886.995386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076886.995456:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076887.123458:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076887.123469:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076887.123475:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076887.123486:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3876 00000100:00080000:1.0:1769076887.123502:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076887.123511:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076887.123564:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617319680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076887.123597:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076887.123602:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076887.123604:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076887.123610:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3876 00000100:00080000:1.0:1769076887.123634:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076887.123643:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076887.123653:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076887.123670:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617319808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076887.123691:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076887.123699:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769076887.123699:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617319680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076887.123704:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076887.123719:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3876 00000100:00080000:1.0:1769076887.123724:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076887.123729:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076887.123737:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076887.123755:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617319936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076887.123777:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076887.123781:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076887.123784:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769076887.123792:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076887.123797:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3876 00000100:00080000:1.0:1769076887.123805:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076887.123810:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076887.123816:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076887.123830:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617320064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076887.123855:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076887.123858:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076887.123864:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076887.123864:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076887/real 1769076887] req@ffff9b0343def100 x1855007617319680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076887.123875:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3876 00000100:00080000:1.0:1769076887.123883:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076887.123888:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076887.123893:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076887.123896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617319680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076887.123907:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617320192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076887.123909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076887.123921:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076887.123928:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617319680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076887.123969:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076887.123993:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076887.124002:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076887/real 1769076887] req@ffff9b0343dedf80 x1855007617319808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076887.124012:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617319808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076887.124020:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076887.124023:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076887.124036:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076887.124042:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076887.124058:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076887.124067:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076887/real 1769076887] req@ffff9b0343ded880 x1855007617319936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076887.124077:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617319936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076887.124084:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076887.124087:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076887.124090:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617319936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076887.124104:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076887.124119:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076887.124129:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076887/real 1769076887] req@ffff9b0343def800 x1855007617320064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076887.124140:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617320064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076887.124147:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076887.124151:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076887.124155:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076887.124159:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076887.124174:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076887.124190:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076887/real 1769076887] req@ffff9b0343deea00 x1855007617320192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076887.124203:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617320192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076887.124211:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076887.124218:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076887.124224:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076887.635391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076888.019357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076888.019402:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076888.019481:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076888.659364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076889.043397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076889.043423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076889.043500:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076889.683451:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076890.067402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076890.067411:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076890.067604:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076890.707421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076891.091339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076891.091354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076891.091527:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076891.731405:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076892.115395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076892.115404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076892.115600:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076892.243434:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076892.243453:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076892.243460:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076892.243475:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3881 00000100:00080000:0.0:1769076892.243491:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076892.243503:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076892.243616:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617320320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076892.243667:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076892.243674:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076892.243678:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076892.243696:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3881 00000100:00080000:0.0:1769076892.243704:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076892.243713:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076892.243728:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076892.243746:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617320320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076892.243762:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617320448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076892.243802:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076892.243809:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076892.243813:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076892.243820:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3881 00000100:00080000:0.0:1769076892.243830:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076892.243837:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769076892.243844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769076892.243852:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076892.243877:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617320576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076892.243896:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076892/real 1769076892] req@ffff9b0365a38a80 x1855007617320320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076892.243898:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076892.243916:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076892.243925:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076892.243936:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3881 00000100:00100000:2.0:1769076892.243940:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617320320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076892.243945:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076892.243961:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769076892.243963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076892.243976:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000008:00100000:0.0:1769076892.243979:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076892.243982:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617320320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076892.244005:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617320704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076892.244018:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320448:192.168.204.160@tcp:38:kworker.0 00000100:00080000:0.0:1769076892.244051:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769076892.244058:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076892.244063:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076892.244070:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076892.244079:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076892/real 1769076892] req@ffff9b0365a39c00 x1855007617320448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076892.244081:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3881 00000100:00080000:0.0:1769076892.244094:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076892.244109:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617320448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076892.244119:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769076892.244126:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769076892.244133:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769076892.244135:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076892.244145:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076892.244156:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769076892.244165:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617320832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076892.244182:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076892.244202:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076892/real 1769076892] req@ffff9b0365a38e00 x1855007617320576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076892.244215:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617320576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076892.244225:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076892.244228:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076892.244265:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076892.244282:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076892.244304:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076892.244315:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076892/real 1769076892] req@ffff9b0365a3b480 x1855007617320704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076892.244336:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617320704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076892.244353:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076892.244356:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076892.244360:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076892.244365:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076892.244380:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076892.244389:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076892/real 1769076892] req@ffff9b0365a3aa00 x1855007617320832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076892.244400:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617320832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076892.244409:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076892.244412:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076892.244415:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617320832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076892.755409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076893.139370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076893.139622:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076893.139652:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076893.779398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076894.163355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076894.163426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076894.163536:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076894.803391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076895.187456:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076895.187596:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076895.187636:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076895.827335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076896.211598:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076896.211680:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076896.211880:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076896.851344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076897.235363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076897.235470:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076897.235622:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076897.363359:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076897.363369:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076897.363373:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076897.363385:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3886 00000100:00080000:0.0:1769076897.363398:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076897.363413:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076897.363510:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617320960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076897.363541:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076897.363545:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076897.363547:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076897.363551:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3886 00000100:00080000:0.0:1769076897.363555:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076897.363559:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076897.363576:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076897.363591:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617321088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076897.363602:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076897.363606:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076897.363608:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076897.363611:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3886 00000100:00080000:0.0:1769076897.363614:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076897.363618:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076897.363623:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076897.363632:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617321216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076897.363657:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617320960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076897.363661:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076897.363679:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076897.363682:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076897.363687:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3886 00000100:00080000:0.0:1769076897.363689:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076897.363692:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076897.363699:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076897.363716:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617321344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076897.363723:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076897.363726:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076897.363729:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076897.363733:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3886 00000100:00080000:0.0:1769076897.363737:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076897.363741:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076897.363746:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076897.363754:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617321472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076897.363786:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076897.363848:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076897/real 1769076897] req@ffff9b0365a3bb80 x1855007617320960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076897.363872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617320960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076897.363886:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076897.363893:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076897.363898:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617320960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076897.363907:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076897.363931:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076897.363944:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076897/real 1769076897] req@ffff9b0365a3b100 x1855007617321088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076897.363972:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617321088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076897.363978:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076897.363980:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076897.363983:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076897.364008:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076897.364020:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076897.364026:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076897/real 1769076897] req@ffff9b0365a39500 x1855007617321216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076897.364034:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617321216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076897.364045:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076897.364047:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076897.364049:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076897.364055:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076897.364075:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076897.364086:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076897/real 1769076897] req@ffff9b0365a39180 x1855007617321344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076897.364099:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617321344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076897.364108:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076897.364112:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076897.364116:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076897.364123:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076897.364139:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076897.364147:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076897/real 1769076897] req@ffff9b0365a3ad80 x1855007617321472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076897.364155:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617321472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076897.364161:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076897.364163:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076897.364166:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076897.875340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076898.259349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076898.259362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076898.259405:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076898.899337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076899.283360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076899.283397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076899.283538:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076899.923454:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076900.307351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076900.307525:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076900.307526:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076900.947321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076901.331304:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076901.331338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076901.331409:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076901.971321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076902.355347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769076902.355347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076902.355420:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076902.483349:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076902.483370:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076902.483374:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076902.483382:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3891 00000100:00080000:2.0:1769076902.483391:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076902.483396:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076902.483439:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617321600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076902.483467:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076902.483470:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076902.483471:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076902.483474:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3891 00000100:00080000:2.0:1769076902.483476:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076902.483479:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076902.483504:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076902.483531:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617321728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076902.483540:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076902.483544:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076902.483546:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076902.483549:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3891 00000100:00080000:2.0:1769076902.483552:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076902.483559:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076902.483561:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617321600:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769076902.483563:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076902.483576:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617321856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076902.483593:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076902.483596:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076902.483598:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076902.483600:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3891 00000100:00080000:2.0:1769076902.483603:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076902.483605:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076902.483609:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076902.483618:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617321984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076902.483624:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076902.483627:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076902.483629:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076902.483632:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3891 00000100:00080000:2.0:1769076902.483635:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076902.483637:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076902.483641:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076902.483643:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076902.483654:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617322112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076902.483692:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076902/real 1769076902] req@ffff9b03482c6680 x1855007617321600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076902.483723:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617321600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076902.483730:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076902.483733:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076902.483736:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617321600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076902.483742:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076902.483789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076902.483802:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076902/real 1769076902] req@ffff9b03482c5880 x1855007617321728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076902.483828:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617321728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076902.483839:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076902.483843:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076902.483848:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076902.483874:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076902.483897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076902.483909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076902/real 1769076902] req@ffff9b03482c7b80 x1855007617321856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076902.483922:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617321856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076902.483931:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076902.483935:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076902.483939:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076902.483946:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076902.483965:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076902.483986:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076902/real 1769076902] req@ffff9b03482c5f80 x1855007617321984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076902.483999:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617321984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076902.484008:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076902.484011:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076902.484015:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617321984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076902.484021:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617322112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076902.484045:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076902.484054:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076902/real 1769076902] req@ffff9b03482c5c00 x1855007617322112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076902.484067:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617322112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076902.484083:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076902.484087:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076902.484091:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617322112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076902.995323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076903.379338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076903.379422:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076903.379494:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076904.019429:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076904.403463:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076904.403516:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769076904.403518:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076905.043447:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769076905.427389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076905.427391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076905.427550:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076906.067470:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076906.451425:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076906.451457:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076906.451501:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076907.091438:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076907.475335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769076907.475342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076907.475430:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076907.603373:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076907.603384:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076907.603406:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076907.603418:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3896 00000100:00080000:0.0:1769076907.603431:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076907.603440:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076907.603484:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617322240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076907.603537:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076907.603541:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076907.603543:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076907.603547:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3896 00000100:00080000:0.0:1769076907.603550:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076907.603554:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076907.603564:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076907.603594:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617322240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076907.603602:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617322368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076907.603623:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076907.603629:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076907.603633:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076907.603640:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3896 00000100:00080000:0.0:1769076907.603668:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076907.603676:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769076907.603683:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769076907.603688:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076907.603708:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617322496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076907.603718:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076907.603723:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076907.603725:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076907.603730:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3896 00000100:00080000:0.0:1769076907.603737:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076907.603737:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076907/real 1769076907] req@ffff9b0365a3ad80 x1855007617322240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076907.603746:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076907.603755:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076907.603773:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617322624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076907.603785:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769076907.603786:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617322240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076907.603791:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076907.603796:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076907.603823:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3896 00000100:00080000:0.0:1769076907.603827:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076907.603828:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769076907.603831:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:1.0:1769076907.603835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000008:00100000:0.0:1769076907.603836:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076907.603841:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617322240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076907.603846:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617322752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076907.603867:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617322368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076907.603894:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076907.603908:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076907/real 1769076907] req@ffff9b0365a39180 x1855007617322368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076907.603920:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617322368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076907.603927:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076907.603929:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076907.603932:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617322368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076907.603936:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617322496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076907.603950:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076907.603956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076907/real 1769076907] req@ffff9b0365a39500 x1855007617322496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076907.603976:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617322496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076907.603981:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076907.603983:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076907.603986:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617322496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076907.603990:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617322624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076907.604009:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076907.604015:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076907/real 1769076907] req@ffff9b0365a3b100 x1855007617322624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076907.604023:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617322624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076907.604029:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076907.604032:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076907.604034:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617322624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076907.604037:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617322752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076907.604047:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076907.604052:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076907/real 1769076907] req@ffff9b0365a3bb80 x1855007617322752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076907.604059:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617322752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076907.604064:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076907.604067:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076907.604085:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617322752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076908.115391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076908.499341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076908.499395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769076908.499401:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076909.139379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076909.523337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076909.523387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769076909.523402:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076910.163509:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076910.547419:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076910.547466:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076910.547490:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076911.187361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076911.571397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076911.571399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076911.571517:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076912.211337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076912.595338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076912.595343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076912.595358:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076912.723373:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076912.723382:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076912.723386:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076912.723394:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3901 00000100:00080000:1.0:1769076912.723404:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076912.723411:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076912.723449:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617322880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076912.723472:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076912.723475:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076912.723476:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076912.723480:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3901 00000100:00080000:1.0:1769076912.723482:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076912.723485:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076912.723492:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076912.723501:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617323008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076912.723510:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076912.723512:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076912.723514:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076912.723517:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3901 00000100:00080000:1.0:1769076912.723519:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076912.723522:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076912.723526:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076912.723532:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617323136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076912.723543:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076912.723546:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076912.723549:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076912.723549:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617322880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076912.723563:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3901 00000100:00080000:1.0:1769076912.723577:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076912.723580:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076912.723583:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076912.723595:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617323264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076912.723602:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076912.723605:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076912.723606:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076912.723608:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3901 00000100:00080000:1.0:1769076912.723612:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076912.723615:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076912.723618:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076912.723627:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617323392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076912.723705:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076912.723750:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076912.723791:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076912.723813:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076913.235308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076913.619319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076913.619363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076913.619466:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076914.259334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076914.643320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076914.643324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076914.643401:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076915.283317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076915.667312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076915.667319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076915.667384:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769076915.795692:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769076915.795733:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076915.795751:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076915.795808:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076915.795816:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076915.795822:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076915.795829:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076915.795869:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076912/real 1769076915] req@ffff9b0343dece00 x1855007617323136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076915.795894:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617323136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076915.795905:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076915.795909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076915.795912:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076915.795918:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076912/real 1769076915] req@ffff9b0343defb80 x1855007617323264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076915.795927:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617323264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076915.795932:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076915.795934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076915.795937:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076915.795941:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076912/real 1769076915] req@ffff9b0343dee680 x1855007617323392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076915.795948:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617323392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076915.795953:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076915.795955:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076915.795958:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076915.795963:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076912/real 1769076915] req@ffff9b0343deed80 x1855007617322880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076915.795975:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617322880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076915.795984:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076915.795988:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076915.795992:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617322880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076915.796000:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076912/real 1769076915] req@ffff9b0343deca80 x1855007617323008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076915.796014:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617323008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076915.796026:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076915.796031:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076915.796036:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076916.307317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076916.691377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076916.691380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076916.691493:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076917.331341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076917.715346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076917.715390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076917.715445:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076917.843405:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076917.843415:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076917.843419:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076917.843446:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3907 00000100:00080000:3.0:1769076917.843465:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076917.843476:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076917.843508:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617323520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076917.843540:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076917.843544:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076917.843545:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076917.843548:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3907 00000100:00080000:3.0:1769076917.843551:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076917.843554:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076917.843562:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076917.843571:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617323648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076917.843580:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076917.843583:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076917.843584:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076917.843591:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3907 00000100:00100000:1.0:1769076917.843593:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617323520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076917.843594:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076917.843598:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076917.843602:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076917.843614:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617323776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076917.843621:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076917.843624:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076917.843625:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076917.843627:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3907 00000100:00080000:3.0:1769076917.843629:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076917.843632:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076917.843635:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076917.843644:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617323904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076917.843649:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076917.843652:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076917.843653:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076917.843657:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3907 00000100:00080000:3.0:1769076917.843660:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076917.843663:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076917.843668:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076917.843671:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076917.843686:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617324032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076917.843724:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076917/real 1769076917] req@ffff9b0352a18700 x1855007617323520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076917.843747:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617323520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076917.843759:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076917.843765:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076917.843770:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617323520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076917.843794:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076917.843820:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076917.843834:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076917/real 1769076917] req@ffff9b0352a1aa00 x1855007617323648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076917.843846:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617323648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076917.843856:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076917.843860:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076917.843864:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076917.843871:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076917.843912:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076917.843925:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076917/real 1769076917] req@ffff9b0352a18e00 x1855007617323776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076917.843937:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617323776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076917.843946:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076917.843950:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076917.843953:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076917.843961:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076917.843984:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076917.843996:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076917/real 1769076917] req@ffff9b0352a1a300 x1855007617323904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076917.844010:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617323904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076917.844019:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076917.844022:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076917.844026:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617323904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076917.844032:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076917.844053:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076917.844064:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076917/real 1769076917] req@ffff9b0352a19f80 x1855007617324032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076917.844078:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617324032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076917.844087:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076917.844090:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076917.844093:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076918.355302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076918.739389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076918.739482:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076918.739489:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076919.379401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076919.763313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076919.763371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076919.763431:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076920.403316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076920.787335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076920.787336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076920.787405:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076921.427385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076921.811349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076921.811349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076921.811415:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076922.451329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076922.835323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076922.835323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076922.835476:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076922.963472:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076922.963483:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076922.963489:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076922.963519:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3912 00000100:00080000:0.0:1769076922.963529:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076922.963537:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076922.963597:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617324160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076922.963642:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076922.963646:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076922.963648:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076922.963652:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3912 00000100:00080000:0.0:1769076922.963657:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076922.963660:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076922.963671:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076922.963680:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617324160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076922.963687:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617324288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076922.963697:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076922.963701:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076922.963703:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076922.963706:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3912 00000100:00080000:0.0:1769076922.963709:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076922.963728:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076922.963732:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076922.963742:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617324416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076922.963750:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076922.963753:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076922.963754:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076922.963758:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3912 00000100:00080000:0.0:1769076922.963762:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076922.963765:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076922.963769:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076922.963778:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617324544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076922.963785:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076922.963791:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076922.963795:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076922.963797:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076922.963801:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3912 00000100:00080000:0.0:1769076922.963811:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076922.963813:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076922.963818:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076922.963829:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617324672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076922.963912:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076922/real 1769076922] req@ffff9b0365a3aa00 x1855007617324160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076922.964070:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617324160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076922.964085:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076922.964090:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076922.964145:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617324160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076922.964179:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076922.964207:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076922.964219:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076922/real 1769076922] req@ffff9b0365a3b480 x1855007617324288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076922.964230:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617324288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076922.964238:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076922.964241:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076922.964244:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076922.964301:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076922.964320:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076922.964330:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076922/real 1769076922] req@ffff9b0365a38e00 x1855007617324416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076922.964357:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617324416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076922.964364:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076922.964367:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076922.964371:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076922.964377:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076922.964396:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076922.964406:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076922/real 1769076922] req@ffff9b0365a39c00 x1855007617324544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076922.964415:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617324544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076922.964421:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076922.964423:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076922.964425:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076922.964429:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076922.964438:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076922.964444:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076922/real 1769076922] req@ffff9b0365a38a80 x1855007617324672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076922.964459:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617324672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076922.964465:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076922.964467:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076922.964469:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076923.475343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076923.859347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076923.859354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076923.859450:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076924.499351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076924.883390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076924.883404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076924.883573:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076925.523356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076925.907385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076925.907385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076925.907516:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076926.547368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076926.931323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076926.931323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076926.931560:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076927.571318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076927.955450:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076927.955486:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076927.955539:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076928.083447:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076928.083461:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076928.083466:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076928.083478:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3917 00000100:00080000:0.0:1769076928.083491:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076928.083501:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076928.083557:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617324800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076928.083606:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076928.083614:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076928.083619:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076928.083631:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3917 00000100:00080000:0.0:1769076928.083638:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076928.083644:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076928.083656:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076928.083669:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617324800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076928.083684:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617324928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076928.083702:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076928.083707:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076928.083710:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076928.083716:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3917 00000100:00080000:0.0:1769076928.083722:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076928.083727:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076928.083733:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076928.083750:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617325056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076928.083761:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076928.083765:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076928.083769:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076928.083775:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3917 00000100:00080000:0.0:1769076928.083779:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076928.083783:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076928.083802:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076928.083815:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617325184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076928.083826:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076928.083841:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076928.083846:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076928.083849:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076928.083855:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3917 00000100:00080000:0.0:1769076928.083861:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076928.083867:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076928.083874:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076928.083888:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617325312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076928.083914:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076928/real 1769076928] req@ffff9b0365a38a80 x1855007617324800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076928.083954:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617324800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076928.083983:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076928.083989:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076928.083993:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617324800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076928.084025:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076928.084070:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076928.084087:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076928/real 1769076928] req@ffff9b0365a39c00 x1855007617324928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076928.084103:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617324928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076928.084112:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076928.084115:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076928.084118:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617324928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076928.084124:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076928.084143:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076928.084152:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076928/real 1769076928] req@ffff9b0365a38e00 x1855007617325056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076928.084166:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617325056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076928.084175:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076928.084178:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076928.084182:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076928.084191:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076928.084216:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076928.084228:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076928/real 1769076928] req@ffff9b0365a3b480 x1855007617325184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076928.084237:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617325184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076928.084248:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076928.084250:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076928.084253:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076928.084258:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076928.084314:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076928.084322:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076928/real 1769076928] req@ffff9b0365a3aa00 x1855007617325312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076928.084331:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617325312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076928.084353:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076928.084355:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076928.084358:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076928.595363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076928.979324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076928.979369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076928.979440:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076929.619403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076930.003406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076930.003407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076930.003483:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076930.643435:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076931.027383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076931.027404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076931.027615:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076931.667337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076932.051345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076932.051349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076932.051424:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076932.691359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076933.075356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076933.075419:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076933.075514:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076933.203342:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076933.203358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076933.203364:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076933.203378:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3922 00000100:00080000:0.0:1769076933.203390:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076933.203400:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076933.203451:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617325440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076933.203486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076933.203491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076933.203493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076933.203498:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3922 00000100:00080000:0.0:1769076933.203503:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076933.203508:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076933.203521:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076933.203536:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617325568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076933.203547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076933.203551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076933.203553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076933.203558:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3922 00000100:00080000:0.0:1769076933.203562:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076933.203567:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076933.203575:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617325440:192.168.204.160@tcp:250:kworker.0 00000008:00100000:0.0:1769076933.203577:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076933.203607:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617325696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076933.203619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076933.203624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076933.203627:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076933.203631:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3922 00000100:00080000:0.0:1769076933.203637:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076933.203641:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076933.203647:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076933.203659:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617325824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076933.203668:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769076933.203674:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076933.203675:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076933.203680:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076933.203685:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3922 00000100:00080000:0.0:1769076933.203692:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076933.203696:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076933.203702:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076933.203716:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617325952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076933.203741:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076933/real 1769076933] req@ffff9b0365a3bb80 x1855007617325440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076933.203762:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617325440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076933.203776:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076933.203780:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076933.203783:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617325440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076933.203789:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076933.203809:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076933.203818:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076933/real 1769076933] req@ffff9b0365a3b100 x1855007617325568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076933.203826:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617325568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076933.203832:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076933.203835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076933.203837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076933.203874:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076933.203887:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076933.203894:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076933/real 1769076933] req@ffff9b0365a39500 x1855007617325696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076933.203902:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617325696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076933.203908:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076933.203910:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076933.203912:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076933.203916:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076933.203927:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076933.203932:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076933/real 1769076933] req@ffff9b0365a39180 x1855007617325824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076933.203940:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617325824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076933.203945:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076933.203946:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076933.203948:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076933.203951:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076933.203960:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076933.203965:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076933/real 1769076933] req@ffff9b0365a3ad80 x1855007617325952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076933.203972:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617325952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076933.203977:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076933.203978:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076933.203980:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617325952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076933.715356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076934.099338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076934.099371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076934.099449:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076934.739401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076935.123343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076935.123344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076935.123446:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076935.763376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076936.147371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076936.147372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076936.147452:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076936.787362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076937.171344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076937.171345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076937.171433:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076937.811328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076938.195333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076938.195369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076938.195453:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076938.323409:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076938.323422:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076938.323427:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076938.323437:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3927 00000100:00080000:2.0:1769076938.323448:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076938.323456:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076938.323496:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617326080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076938.323526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076938.323530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076938.323533:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076938.323538:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3927 00000100:00080000:2.0:1769076938.323543:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076938.323548:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076938.323559:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076938.323583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617326208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076938.323585:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617326080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076938.323613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076938.323618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076938.323621:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076938.323628:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3927 00000100:00080000:2.0:1769076938.323633:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076938.323637:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076938.323643:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076938.323658:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617326336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076938.323669:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076938.323672:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076938.323674:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076938.323679:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3927 00000100:00080000:2.0:1769076938.323684:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076938.323688:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076938.323695:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769076938.323704:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076938.323711:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617326464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076938.323725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076938.323729:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076938.323732:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076938.323737:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3927 00000100:00080000:2.0:1769076938.323741:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076938.323746:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076938.323751:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076938.323777:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617326592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076938.323783:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076938/real 1769076938] req@ffff9b03482c4e00 x1855007617326080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076938.323837:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617326080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076938.323849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076938.323855:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076938.323860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617326080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076938.323892:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076938.323920:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076938.323931:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076938/real 1769076938] req@ffff9b03482c5c00 x1855007617326208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076938.323941:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617326208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076938.323948:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076938.323951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076938.323953:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076938.323959:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076938.323971:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076938.323978:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076938/real 1769076938] req@ffff9b03482c5f80 x1855007617326336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076938.323987:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617326336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076938.323995:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076938.323998:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076938.324002:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076938.324009:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076938.324031:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076938.324043:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076938/real 1769076938] req@ffff9b03482c7b80 x1855007617326464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076938.324056:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617326464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076938.324062:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076938.324064:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076938.324066:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076938.324072:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076938.324086:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076938.324093:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076938/real 1769076938] req@ffff9b03482c5880 x1855007617326592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076938.324101:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617326592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076938.324112:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076938.324114:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076938.324116:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076938.835409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076939.219317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076939.219366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076939.219431:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076939.859355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076940.243375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076940.243375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076940.243489:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076940.883402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076941.267332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076941.267351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076941.267480:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076941.907437:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076942.291468:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076942.291469:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076942.291633:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076942.931475:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076943.315422:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076943.315427:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076943.315457:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076943.443461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076943.443477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076943.443485:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076943.443501:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3932 00000100:00080000:0.0:1769076943.443518:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076943.443559:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076943.443618:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617326720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076943.443657:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076943.443662:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076943.443665:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076943.443705:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3932 00000100:00080000:0.0:1769076943.443714:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076943.443722:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076943.443735:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076943.443740:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617326720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076943.443759:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617326848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076943.443777:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076943.443783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076943.443786:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076943.443802:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3932 00000100:00080000:0.0:1769076943.443806:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076943.443811:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076943.443817:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076943.443832:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617326976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076943.443852:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076943.443873:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076943.443881:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076943.443886:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076943.443895:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3932 00000100:00080000:0.0:1769076943.443905:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769076943.443916:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076943/real 1769076943] req@ffff9b0365a3ad80 x1855007617326720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769076943.443918:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076943.443940:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076943.443955:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617326720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076943.443972:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769076943.443974:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617327104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076943.443982:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076943.443996:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617326720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769076943.444020:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076943.444028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076943.444033:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769076943.444037:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326848:192.168.204.160@tcp:38:kworker.0 00000100:00080000:0.0:1769076943.444048:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3932 00000100:00080000:0.0:1769076943.444063:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076943.444073:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769076943.444082:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769076943.444087:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076943.444102:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076943/real 1769076943] req@ffff9b0365a39180 x1855007617326848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076943.444120:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617327232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769076943.444125:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617326848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076943.444181:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076943.444191:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076943.444197:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076943.444208:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076943.444267:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076943.444282:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076943/real 1769076943] req@ffff9b0365a39500 x1855007617326976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076943.444297:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617326976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076943.444317:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076943.444321:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076943.444325:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617326976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076943.444332:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076943.444355:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076943.444366:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076943/real 1769076943] req@ffff9b0365a3b100 x1855007617327104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076943.444379:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617327104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076943.444389:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076943.444392:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076943.444408:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076943.444427:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076943.444446:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076943.444457:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076943/real 1769076943] req@ffff9b0365a3bb80 x1855007617327232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076943.444471:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617327232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076943.444480:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076943.444484:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076943.444488:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076943.955474:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076944.339374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076944.339393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076944.339649:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076944.979426:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076945.363362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076945.363393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076945.363582:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076946.003347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076946.387384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076946.387394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076946.387495:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076947.027376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076947.411369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076947.411377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076947.411493:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076948.051411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076948.435337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076948.435359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076948.435530:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769076948.563392:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076948.563418:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076948.563425:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076948.563437:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3937 00000100:00080000:0.0:1769076948.563461:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076948.563470:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076948.563516:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617327360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076948.563552:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076948.563566:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076948.563569:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076948.563575:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3937 00000100:00080000:0.0:1769076948.563580:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076948.563609:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076948.563624:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076948.563626:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617327360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076948.563649:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617327488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076948.563665:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076948.563670:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076948.563673:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076948.563678:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3937 00000100:00080000:0.0:1769076948.563685:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076948.563689:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076948.563696:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076948.563711:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617327616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769076948.563719:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769076948.563734:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076948.563740:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076948.563743:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076948.563750:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3937 00000100:00080000:0.0:1769076948.563754:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076948.563758:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076948.563766:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076948.563777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076948/real 1769076948] req@ffff9b0365a3aa00 x1855007617327360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076948.563786:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617327744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076948.563824:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076948.563836:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076948.563838:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076948.563844:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3937 00000100:00080000:0.0:1769076948.563849:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076948.563855:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076948.563861:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076948.563864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617327360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769076948.563877:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617327872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076948.563909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076948.563920:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076948.563930:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617327360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076948.563968:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076948.564010:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076948.564028:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076948/real 1769076948] req@ffff9b0365a3b480 x1855007617327488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076948.564045:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617327488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076948.564055:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076948.564060:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076948.564064:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076948.564072:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769076948.564091:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076948.564114:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076948/real 1769076948] req@ffff9b0365a38e00 x1855007617327616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076948.564128:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617327616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076948.564137:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076948.564141:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076948.564144:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076948.564161:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076948.564179:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076948.564191:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076948/real 1769076948] req@ffff9b0365a39c00 x1855007617327744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076948.564205:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617327744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076948.564214:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076948.564217:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076948.564221:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076948.564228:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769076948.564278:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076948.564291:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076948/real 1769076948] req@ffff9b0365a38a80 x1855007617327872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769076948.564305:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617327872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076948.564314:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769076948.564318:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769076948.564321:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617327872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076949.075335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076949.459332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076949.459333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076949.459421:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076950.099374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076950.483372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076950.483372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076950.483424:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076951.123375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076951.507338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076951.507340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076951.507403:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076952.147366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076952.531337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076952.531438:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076952.531651:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076953.171400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076953.555442:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076953.555461:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076953.555492:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076953.683454:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076953.683474:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076953.683482:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076953.683523:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3942 00000100:00080000:3.0:1769076953.683541:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076953.683553:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076953.683620:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617328000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076953.683670:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076953.683675:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076953.683677:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076953.683682:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3942 00000100:00080000:3.0:1769076953.683686:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076953.683690:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076953.683702:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076953.683716:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617328128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076953.683727:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076953.683731:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076953.683733:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076953.683737:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3942 00000100:00080000:3.0:1769076953.683741:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076953.683748:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617328000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076953.683749:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076953.683759:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076953.683771:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617328256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076953.683781:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076953.683786:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076953.683788:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076953.683792:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3942 00000100:00080000:3.0:1769076953.683795:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076953.683799:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076953.683804:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076953.683815:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617328384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076953.683834:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076953.683837:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076953.683839:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076953.683844:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3942 00000100:00080000:3.0:1769076953.683848:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076953.683850:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076953.683855:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769076953.683863:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076953.683872:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617328512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076953.683938:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076953/real 1769076953] req@ffff9b0352a19f80 x1855007617328000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076953.683960:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617328000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076953.683977:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076953.683984:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076953.683991:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617328000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076953.684013:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076953.684049:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076953.684068:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076953/real 1769076953] req@ffff9b0352a1a300 x1855007617328128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076953.684085:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617328128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076953.684097:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076953.684102:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076953.684107:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076953.684144:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076953.684176:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076953.684194:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076953/real 1769076953] req@ffff9b0352a18e00 x1855007617328256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076953.684212:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617328256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076953.684265:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076953.684269:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076953.684273:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076953.684284:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076953.684318:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076953.684356:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076953/real 1769076953] req@ffff9b0352a1aa00 x1855007617328384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076953.684375:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617328384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076953.684399:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076953.684403:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076953.684407:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076953.684413:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076953.684435:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076953.684445:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076953/real 1769076953] req@ffff9b0352a18700 x1855007617328512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076953.684466:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617328512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076953.684473:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076953.684476:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076953.684479:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076954.195418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076954.579394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076954.579394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076954.579475:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076955.219361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076955.603379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076955.603380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076955.603539:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076956.243402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076956.627337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769076956.627444:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076956.627564:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076957.267365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076957.651351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076957.651377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076957.651470:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076958.291308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076958.675320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076958.675462:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076958.675564:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769076958.803381:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076958.803395:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076958.803401:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076958.803437:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3947 00000100:00080000:3.0:1769076958.803450:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076958.803458:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076958.803517:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617328640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076958.803551:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076958.803555:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076958.803556:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076958.803559:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3947 00000100:00080000:3.0:1769076958.803565:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076958.803581:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076958.803594:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076958.803607:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617328640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076958.803614:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617328768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076958.803623:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076958.803627:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076958.803628:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076958.803631:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3947 00000100:00080000:3.0:1769076958.803634:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076958.803637:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076958.803641:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076958.803649:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617328896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076958.803661:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076958.803664:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076958.803666:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076958.803669:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3947 00000100:00080000:3.0:1769076958.803678:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076958.803682:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076958.803687:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769076958.803700:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769076958.803703:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617329024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076958.803717:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076958.803721:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076958.803724:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076958.803731:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3947 00000100:00080000:3.0:1769076958.803736:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076958.803740:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076958.803743:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076958.803761:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617329152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076958.803777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076958/real 1769076958] req@ffff9b0352a1a680 x1855007617328640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076958.803799:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617328640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076958.803809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076958.803816:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076958.803823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617328640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076958.803862:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076958.803880:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076958.803889:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076958/real 1769076958] req@ffff9b0352a18a80 x1855007617328768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076958.803899:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617328768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076958.803918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076958.803920:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076958.803923:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076958.803927:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076958.803938:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076958.803945:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076958/real 1769076958] req@ffff9b0352a1b480 x1855007617328896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076958.803953:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617328896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076958.803959:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076958.803962:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076958.803964:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617328896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076958.803968:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076958.803980:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076958.803986:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076958/real 1769076958] req@ffff9b0352a1ad80 x1855007617329024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076958.803993:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617329024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076958.803998:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076958.804000:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076958.804002:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076958.804006:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076958.804015:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076958.804023:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076958/real 1769076958] req@ffff9b0352a1b100 x1855007617329152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076958.804030:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617329152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076958.804036:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076958.804038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076958.804039:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076959.315369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769076959.699533:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769076959.699626:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076959.699672:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076960.339449:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076960.723800:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769076960.723853:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076960.723878:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076961.363446:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076961.747465:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769076961.747472:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076961.747636:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076962.387450:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076962.771321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076962.771428:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076962.771482:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076963.411360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076963.795381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076963.795416:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076963.795508:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:0.0:1769076963.923317:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076963.923339:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769076963.923343:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076963.923351:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3953 00000100:00080000:0.0:1769076963.923378:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076963.923386:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769076963.923421:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617329280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076963.923450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076963.923454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769076963.923456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076963.923460:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3953 00000100:00080000:0.0:1769076963.923462:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076963.923466:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076963.923474:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769076963.923502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617329408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076963.923510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076963.923513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769076963.923514:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076963.923517:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3953 00000100:00080000:0.0:1769076963.923519:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076963.923522:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076963.923526:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076963.923531:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617329280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769076963.923535:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617329536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076963.923550:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076963.923576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769076963.923578:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076963.923581:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3953 00000100:00080000:0.0:1769076963.923585:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076963.923588:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769076963.923594:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076963.923604:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617329664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769076963.923613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769076963.923616:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769076963.923633:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769076963.923636:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3953 00000100:00080000:0.0:1769076963.923638:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769076963.923643:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769076963.923646:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769076963.923648:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769076963.923665:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617329792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076963.923707:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076963/real 1769076963] req@ffff9b0365a38a80 x1855007617329280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076963.923738:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617329280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076963.923749:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076963.923754:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076963.923758:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617329280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076963.923766:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076963.923791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076963.923804:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076963/real 1769076963] req@ffff9b0365a39c00 x1855007617329408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076963.923817:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617329408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076963.923826:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076963.923829:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076963.923833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076963.923856:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076963.923876:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076963.923887:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076963/real 1769076963] req@ffff9b0365a38e00 x1855007617329536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076963.923906:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617329536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076963.923914:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076963.923917:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076963.923920:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076963.923926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076963.923945:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076963.923955:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076963/real 1769076963] req@ffff9b0365a3b480 x1855007617329664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076963.923967:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617329664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076963.923975:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076963.923979:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076963.923982:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076963.923988:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076963.924005:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076963.924015:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076963/real 1769076963] req@ffff9b0365a3aa00 x1855007617329792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076963.924027:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617329792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076963.924040:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076963.924044:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076963.924047:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617329792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076964.435305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076964.819370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076964.819634:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769076964.820122:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076965.459342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076965.843403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076965.843422:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076965.843516:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076966.483364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076966.867417:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076966.867422:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076966.867524:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076967.507393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076967.891419:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076967.891433:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076967.891467:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076968.531323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076968.915443:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076968.915451:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076968.915477:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076969.043415:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076969.043429:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076969.043435:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076969.043444:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3958 00000100:00080000:3.0:1769076969.043454:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076969.043462:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076969.043503:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617329920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076969.043540:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076969.043546:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076969.043549:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076969.043556:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3958 00000100:00080000:3.0:1769076969.043562:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076969.043568:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076969.043579:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076969.043585:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617329920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076969.043597:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617330048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076969.043610:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076969.043614:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076969.043617:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076969.043620:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3958 00000100:00080000:3.0:1769076969.043624:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076969.043628:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076969.043644:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076969.043653:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617330176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076969.043662:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076969.043664:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076969.043666:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076969.043669:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3958 00000800:00000100:2.0:1769076969.043675:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769076969.043675:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076969.043684:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076969.043689:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076969.043709:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617330304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076969.043717:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076969.043720:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076969.043722:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076969.043727:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3958 00000100:00080000:3.0:1769076969.043731:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769076969.043736:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9b0352a18380 x1855007617329920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076969.043737:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076969.043747:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769076969.043766:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617330432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076969.043770:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617329920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.043794:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.043800:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076969.043804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617329920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076969.043826:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076969.043847:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076969.043858:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9b0352a1b800 x1855007617330048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076969.043896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617330048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.043910:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.043914:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076969.043918:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076969.043926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076969.043952:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076969.043967:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9b0352a19500 x1855007617330176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076969.043983:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617330176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.043992:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.043996:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076969.044001:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076969.044008:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076969.044034:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076969.044047:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9b0352a18000 x1855007617330304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076969.044062:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617330304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.044084:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.044088:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076969.044110:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076969.044121:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076969.044154:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076969.044173:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076969/real 1769076969] req@ffff9b0352a19c00 x1855007617330432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076969.044186:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617330432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076969.044195:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076969.044198:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076969.044201:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076969.555343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076969.939354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076969.939376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076969.939403:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076970.579346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076970.963350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076970.963356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076970.963414:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769076971.603419:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076971.987401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076971.987471:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076971.987539:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076972.627323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076973.011339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076973.011350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076973.011403:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076973.651355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076974.035309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076974.035325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076974.035328:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769076974.163363:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076974.163374:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769076974.163396:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076974.163407:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3963 00000100:00080000:3.0:1769076974.163416:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076974.163423:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769076974.163472:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617330560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076974.163537:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076974.163541:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769076974.163544:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076974.163549:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3963 00000100:00080000:3.0:1769076974.163571:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076974.163578:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076974.163590:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076974.163604:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617330560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076974.163611:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617330688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076974.163624:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076974.163628:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769076974.163638:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076974.163645:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3963 00000100:00080000:3.0:1769076974.163659:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076974.163665:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769076974.163671:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769076974.163674:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769076974.163695:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617330816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769076974.163708:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076974.163712:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076974/real 1769076974] req@ffff9b0352a19c00 x1855007617330560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769076974.163714:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769076974.163718:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076974.163727:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3963 00000100:00080000:3.0:1769076974.163734:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076974.163738:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076974.163753:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617330560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769076974.163754:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769076974.163769:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769076974.163773:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617330944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076974.163777:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076974.163786:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617330560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769076974.163791:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769076974.163811:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769076974.163813:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769076974.163817:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3963 00000100:00080000:3.0:1769076974.163819:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769076974.163822:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769076974.163828:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076974.163830:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769076974.163839:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617331072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076974.163856:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076974.163869:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076974/real 1769076974] req@ffff9b0352a18000 x1855007617330688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076974.163878:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617330688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076974.163884:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076974.163887:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076974.163889:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076974.163893:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076974.163904:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076974.163909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076974/real 1769076974] req@ffff9b0352a19500 x1855007617330816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076974.163917:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617330816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076974.163922:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076974.163924:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076974.163926:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076974.163929:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076974.163941:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076974.163947:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076974/real 1769076974] req@ffff9b0352a1b800 x1855007617330944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076974.163955:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617330944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076974.163960:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076974.163961:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076974.163963:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617330944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076974.163973:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076974.163983:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076974.163993:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076974/real 1769076974] req@ffff9b0352a18380 x1855007617331072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076974.164000:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617331072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769076999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076974.164005:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076974.164007:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076974.164009:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076974.675329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076975.059322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076975.059323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076975.059376:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076975.699359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076976.083330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076976.083330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076976.083391:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076976.723313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076977.107341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076977.107341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076977.107469:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076977.747315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076978.131310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076978.131318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076978.131423:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076978.771393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076979.155340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769076979.155360:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076979.155386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769076979.283364:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076979.283377:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076979.283382:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076979.283393:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3968 00000100:00080000:2.0:1769076979.283415:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076979.283423:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076979.283495:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617331200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076979.283543:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076979.283547:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076979.283548:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076979.283551:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3968 00000100:00080000:2.0:1769076979.283555:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076979.283557:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076979.283566:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076979.283577:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617331328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076979.283602:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076979.283604:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076979.283606:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076979.283611:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3968 00000100:00100000:1.0:1769076979.283613:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617331200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076979.283614:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076979.283622:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076979.283627:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076979.283639:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617331456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076979.283649:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076979.283653:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076979.283655:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076979.283659:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3968 00000100:00080000:2.0:1769076979.283664:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076979.283669:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076979.283675:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076979.283686:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617331584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076979.283696:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076979.283698:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076979.283700:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076979.283703:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3968 00000100:00080000:2.0:1769076979.283706:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076979.283709:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076979.283713:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076979.283743:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617331712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076979.283757:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076979.283815:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076979.283839:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076979.283856:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076979.795328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076980.179420:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076980.179421:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076980.179491:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076980.819290:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076981.203326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076981.203384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076981.203444:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076981.843295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076982.227372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076982.227376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076982.227453:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769076982.355762:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769076982.355822:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769076982.355845:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076982.355890:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076982.355900:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076982.355909:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769076982.355918:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769076982.355963:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076979/real 1769076982] req@ffff9b03482c4700 x1855007617331712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076982.356000:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617331712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076982.356013:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076982.356038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076982.356042:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076982.356051:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076979/real 1769076982] req@ffff9b03482c6a00 x1855007617331328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076982.356066:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617331328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076982.356073:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076982.356076:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076982.356082:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076982.356089:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076979/real 1769076982] req@ffff9b03482c4000 x1855007617331456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076982.356101:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617331456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076982.356110:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076982.356113:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076982.356116:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076982.356123:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076979/real 1769076982] req@ffff9b03482c7100 x1855007617331584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076982.356136:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617331584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076982.356145:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076982.356148:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076982.356152:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076982.356159:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076979/real 1769076982] req@ffff9b03482c6680 x1855007617331200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076982.356172:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617331200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076982.356180:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076982.356184:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076982.356188:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617331200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769076982.867330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076983.251396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076983.251405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076983.251516:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076983.891331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076984.275322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076984.275413:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076984.275420:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076984.403341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076984.403353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076984.403357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076984.403367:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3973 00000100:00080000:2.0:1769076984.403378:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076984.403386:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076984.403426:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617331840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076984.403474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076984.403478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076984.403479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076984.403483:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3973 00000100:00080000:2.0:1769076984.403486:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076984.403489:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076984.403497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076984.403507:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617331968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076984.403529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076984.403531:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076984.403533:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076984.403535:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3973 00000100:00080000:2.0:1769076984.403537:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076984.403539:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076984.403542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076984.403550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617332096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076984.403556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076984.403558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076984.403559:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076984.403562:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3973 00000100:00080000:2.0:1769076984.403564:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076984.403566:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076984.403570:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076984.403574:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617331840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076984.403580:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617332224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076984.403587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076984.403589:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076984.403590:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076984.403593:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3973 00000100:00080000:2.0:1769076984.403595:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076984.403597:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076984.403600:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076984.403610:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617332352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769076984.403647:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076984.403690:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076984/real 1769076984] req@ffff9b03482c6d80 x1855007617331840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076984.403710:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617331840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076984.403720:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076984.403724:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076984.403729:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617331840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076984.403745:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076984.403768:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076984.403781:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076984/real 1769076984] req@ffff9b03482c6300 x1855007617331968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076984.403793:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617331968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076984.403801:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076984.403804:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076984.403807:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617331968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076984.403813:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076984.403834:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076984.403845:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076984/real 1769076984] req@ffff9b03482c5500 x1855007617332096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076984.403856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617332096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076984.403864:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076984.403868:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076984.403871:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076984.403901:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076984.403923:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076984.403933:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076984/real 1769076984] req@ffff9b03482c4a80 x1855007617332224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076984.403945:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617332224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076984.403952:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076984.403956:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076984.403959:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076984.403965:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076984.404001:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076984.404013:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076984/real 1769076984] req@ffff9b03482c5180 x1855007617332352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076984.404032:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617332352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076984.404040:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076984.404043:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076984.404046:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076984.915312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076985.299381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076985.299381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076985.299449:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076985.939299:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076986.323356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076986.323404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076986.323422:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076986.963363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076987.347358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076987.347412:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076987.347521:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076987.987386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076988.371345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076988.371400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076988.371504:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076989.011326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769076989.395433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076989.395493:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769076989.395572:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076989.523433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076989.523450:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076989.523457:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076989.523473:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3978 00000100:00080000:1.0:1769076989.523489:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076989.523500:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076989.523562:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617332480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076989.523609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076989.523615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076989.523617:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076989.523623:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3978 00000100:00080000:1.0:1769076989.523628:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076989.523632:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076989.523659:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076989.523675:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617332608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076989.523693:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769076989.523702:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617332480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076989.523703:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076989.523710:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076989.523725:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3978 00000100:00080000:1.0:1769076989.523729:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076989.523734:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076989.523740:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769076989.523758:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617332736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076989.523772:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076989.523777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076989.523779:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076989.523785:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3978 00000100:00080000:1.0:1769076989.523790:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076989.523794:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076989.523801:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076989.523829:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617332864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769076989.523843:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769076989.523856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076989.523861:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076989.523866:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076989.523874:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3978 00000100:00080000:1.0:1769076989.523883:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076989.523888:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076989.523916:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769076989.523940:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617332992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076989.523950:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076989/real 1769076989] req@ffff9b0343deca80 x1855007617332480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076989.523988:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617332480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076989.524007:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076989.524017:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076989.524025:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617332480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769076989.524072:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076989.524123:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076989.524146:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076989/real 1769076989] req@ffff9b0343deed80 x1855007617332608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076989.524172:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617332608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076989.524193:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076989.524197:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076989.524202:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076989.524210:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076989.524271:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076989.524286:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076989/real 1769076989] req@ffff9b0343dee680 x1855007617332736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076989.524302:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617332736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076989.524312:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076989.524316:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076989.524321:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076989.524328:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076989.524351:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076989.524363:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076989/real 1769076989] req@ffff9b0343defb80 x1855007617332864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076989.524377:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617332864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076989.524396:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076989.524400:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076989.524404:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076989.524410:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076989.524427:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076989.524437:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076989/real 1769076989] req@ffff9b0343dece00 x1855007617332992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076989.524451:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617332992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076989.524461:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076989.524464:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076989.524468:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617332992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076990.035359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076990.419395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076990.419439:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076990.419570:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076991.059421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076991.443320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076991.443320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076991.443419:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076992.083421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076992.467364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076992.467401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769076992.467583:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076993.107386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076993.491415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076993.491519:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076993.491631:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076994.131395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076994.515346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769076994.515347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769076994.515416:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769076994.643366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076994.643377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769076994.643381:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076994.643393:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3983 00000100:00080000:1.0:1769076994.643415:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076994.643424:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769076994.643466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617333120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076994.643508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076994.643514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769076994.643517:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076994.643526:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3983 00000100:00080000:1.0:1769076994.643533:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076994.643541:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076994.643568:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076994.643582:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617333120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076994.643594:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617333248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076994.643616:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076994.643622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769076994.643625:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076994.643633:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3983 00000100:00080000:1.0:1769076994.643639:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076994.643645:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076994.643653:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769076994.643660:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076994.643676:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617333376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769076994.643691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076994.643696:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769076994.643699:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769076994.643708:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3983 00000100:00100000:2.0:1769076994.643709:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076994/real 1769076994] req@ffff9b0343ded500 x1855007617333120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076994.643717:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076994.643729:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076994.643739:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617333120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769076994.643740:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769076994.643760:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769076994.643768:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617333504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076994.643769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076994.643779:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617333120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769076994.643803:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769076994.643826:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769076994.643832:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769076994.643837:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617333248:192.168.204.160@tcp:38:kworker.0 00000100:00080000:1.0:1769076994.643846:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3983 00000100:00080000:1.0:1769076994.643856:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769076994.643863:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769076994.643873:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769076994.643878:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076994.643897:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617333632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769076994.643901:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076994/real 1769076994] req@ffff9b0343def480 x1855007617333248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076994.643944:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617333248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076994.643954:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076994.643958:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076994.643962:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617333248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076994.643970:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617333376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769076994.643991:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076994.644003:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076994/real 1769076994] req@ffff9b0343dece00 x1855007617333376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076994.644017:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617333376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076994.644026:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076994.644030:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076994.644034:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617333376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769076994.644040:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617333504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076994.644061:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076994.644080:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076994/real 1769076994] req@ffff9b0343defb80 x1855007617333504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076994.644101:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617333504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076994.644117:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076994.644125:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076994.644134:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617333504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769076994.644173:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617333632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769076994.644203:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769076994.644219:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076994/real 1769076994] req@ffff9b0343dee680 x1855007617333632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769076994.644285:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617333632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076994.644299:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769076994.644304:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769076994.644310:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617333632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769076995.155363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076995.539316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076995.539335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076995.539522:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076996.179366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076996.563397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076996.563435:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076996.563497:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076997.203386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769076997.587347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076997.587437:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769076997.587437:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076998.227413:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076998.611389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076998.611409:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076998.611458:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769076999.251445:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769076999.635313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769076999.635384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769076999.635504:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769076999.763477:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076999.763490:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769076999.763497:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076999.763510:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3988 00000100:00080000:2.0:1769076999.763522:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076999.763532:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769076999.763579:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617333760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076999.763619:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076999.763624:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769076999.763626:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076999.763632:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3988 00000100:00080000:2.0:1769076999.763636:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076999.763641:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076999.763654:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076999.763672:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617333888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076999.763689:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617333760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769076999.763692:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076999.763706:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769076999.763711:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076999.763718:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3988 00000100:00080000:2.0:1769076999.763754:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076999.763760:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076999.763769:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769076999.763786:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617334016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076999.763799:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076999.763803:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769076999.763805:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076999.763810:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3988 00000100:00080000:2.0:1769076999.763815:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769076999.763820:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769076999.763831:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769076999.763833:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076999.763862:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617334144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769076999.763875:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769076999.763880:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769076999.763882:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769076999.763888:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3988 00000100:00080000:2.0:1769076999.763894:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769076999.763896:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076999/real 1769076999] req@ffff9b03482c4380 x1855007617333760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769076999.763903:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769076999.763916:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769076999.763941:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617334272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769076999.763946:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617333760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076999.763993:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076999.764000:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076999.764007:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617333760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769076999.764048:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617333888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076999.764094:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076999.764120:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076999/real 1769076999] req@ffff9b03482c5180 x1855007617333888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076999.764149:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617333888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076999.764166:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076999.764173:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076999.764180:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617333888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076999.764191:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769076999.764228:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076999.764282:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076999/real 1769076999] req@ffff9b03482c4a80 x1855007617334016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076999.764298:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617334016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076999.764309:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076999.764313:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076999.764317:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769076999.764325:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076999.764350:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076999.764361:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076999/real 1769076999] req@ffff9b03482c5500 x1855007617334144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076999.764375:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617334144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076999.764395:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076999.764401:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076999.764408:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769076999.764420:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769076999.764455:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769076999.764477:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769076999/real 1769076999] req@ffff9b03482c6300 x1855007617334272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769076999.764502:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617334272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769076999.764518:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769076999.764525:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769076999.764532:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077000.275418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077000.659356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077000.659470:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077000.659532:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077001.299397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077001.683340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077001.683396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077001.683408:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077002.323418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077002.707335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077002.707335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077002.707496:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077003.347338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077003.731334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077003.731377:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077003.731415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077004.371375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077004.755410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077004.755423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077004.755551:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077004.883408:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077004.883423:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077004.883430:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077004.883446:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3994 00000100:00080000:3.0:1769077004.883460:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077004.883471:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077004.883529:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617334400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077004.883576:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077004.883581:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077004.883584:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077004.883589:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3994 00000100:00080000:3.0:1769077004.883594:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077004.883599:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077004.883610:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077004.883628:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617334528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077004.883676:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077004.883683:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617334400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077004.883684:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077004.883691:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077004.883703:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3994 00000100:00080000:3.0:1769077004.883709:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077004.883716:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077004.883727:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077004.883773:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617334656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077004.883794:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077004.883801:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077004.883806:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769077004.883835:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077004.883842:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3994 00000100:00080000:3.0:1769077004.883855:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077004.883864:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077004.883873:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077004.883912:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617334784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077004.883927:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077004.883935:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077004.883938:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077004.883945:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3994 00000100:00080000:3.0:1769077004.883951:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077004.883954:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077004/real 1769077004] req@ffff9b0352a1b100 x1855007617334400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077004.883962:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077004.883981:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077004.884003:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617334400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077004.884013:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617334912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077004.884028:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077004.884046:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077004.884057:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617334400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077004.884098:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077004.884178:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077004.884217:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077004/real 1769077004] req@ffff9b0352a1ad80 x1855007617334528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077004.884276:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617334528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077004.884289:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077004.884293:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077004.884297:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077004.884306:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077004.884335:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077004.884363:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077004/real 1769077004] req@ffff9b0352a1b480 x1855007617334656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077004.884377:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617334656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077004.884387:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077004.884391:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077004.884396:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077004.884408:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077004.884443:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077004.884464:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077004/real 1769077004] req@ffff9b0352a18a80 x1855007617334784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077004.884488:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617334784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077004.884504:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077004.884511:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077004.884516:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077004.884524:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077004.884551:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077004.884564:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077004/real 1769077004] req@ffff9b0352a1a680 x1855007617334912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077004.884577:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617334912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077004.884586:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077004.884590:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077004.884593:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617334912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077005.395365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077005.779457:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077005.779478:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077005.779551:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077006.419428:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077006.803358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077006.803389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077006.803528:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077007.443384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077007.827335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077007.827377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077007.827403:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077008.467313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077008.851314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077008.851314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077008.851385:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077009.491324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077009.875415:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077009.875426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077009.875531:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077010.003443:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077010.003457:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077010.003464:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077010.003476:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 3999 00000100:00080000:1.0:1769077010.003489:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077010.003498:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077010.003547:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617335040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077010.003626:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077010.003632:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077010.003634:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077010.003640:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3999 00000100:00080000:1.0:1769077010.003645:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077010.003649:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077010.003660:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077010.003678:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617335168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077010.003696:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077010.003700:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077010.003703:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077010.003712:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617335040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077010.003716:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3999 00000100:00080000:1.0:1769077010.003726:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077010.003733:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077010.003749:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077010.003765:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617335296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077010.003776:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077010.003779:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077010.003782:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077010.003786:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3999 00000100:00080000:1.0:1769077010.003789:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077010.003793:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077010.003799:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077010.003810:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617335424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077010.003820:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077010.003823:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077010.003825:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077010.003847:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 3999 00000100:00080000:1.0:1769077010.003863:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769077010.003872:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077010.003873:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077010.003885:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077010.003899:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617335552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077010.003964:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff9b0343deed80 x1855007617335040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077010.003992:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617335040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077010.004005:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077010.004011:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077010.004017:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617335040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077010.004048:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077010.004078:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077010.004093:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff9b0343deca80 x1855007617335168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077010.004108:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617335168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077010.004119:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077010.004123:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077010.004128:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077010.004135:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077010.004155:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077010.004165:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff9b0343deea00 x1855007617335296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077010.004178:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617335296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077010.004188:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077010.004191:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077010.004195:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077010.004201:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077010.004219:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077010.004229:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff9b0343def800 x1855007617335424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077010.004242:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617335424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077010.004260:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077010.004325:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077010.004330:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077010.004337:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077010.004360:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077010.004373:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077010/real 1769077010] req@ffff9b0343ded880 x1855007617335552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077010.004418:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617335552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077010.004429:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077010.004432:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077010.004436:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077010.515331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077010.899364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077010.899364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077010.899603:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077011.539357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077011.923427:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077011.923427:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077011.923578:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077012.563314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077012.947380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077012.947384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077012.947527:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077013.587365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077013.971358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077013.971361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077013.971452:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077014.611374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077014.995308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077014.995361:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077014.995392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:1.0:1769077015.123350:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077015.123359:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077015.123363:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077015.123371:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4004 00000100:00080000:1.0:1769077015.123380:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077015.123386:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077015.123440:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617335680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077015.123500:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077015.123507:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077015.123510:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077015.123517:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4004 00000100:00080000:1.0:1769077015.123523:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077015.123529:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077015.123539:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617335680:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769077015.123540:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077015.123560:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617335808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077015.123574:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077015.123579:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077015.123581:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077015.123586:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4004 00000100:00080000:1.0:1769077015.123590:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077015.123595:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077015.123601:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077015.123615:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617335936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077015.123615:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077015.123626:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077015.123630:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077015.123633:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077015.123639:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4004 00000100:00080000:1.0:1769077015.123643:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077015.123648:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077015.123654:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077015.123662:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077015/real 1769077015] req@ffff9b0343dedf80 x1855007617335680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077015.123666:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617336064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077015.123679:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077015.123684:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077015.123687:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077015.123687:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617335680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077015.123692:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4004 00000100:00080000:1.0:1769077015.123697:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077015.123701:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077015.123708:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769077015.123715:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769077015.123720:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617336192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077015.123721:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077015.123727:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617335680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077015.123758:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077015.123793:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077015.123811:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077015/real 1769077015] req@ffff9b0343def100 x1855007617335808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077015.123823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617335808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077015.123831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077015.123834:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077015.123838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077015.123843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077015.123861:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077015.123869:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077015/real 1769077015] req@ffff9b0343dee300 x1855007617335936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077015.123880:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617335936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077015.123886:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077015.123889:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077015.123892:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617335936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077015.123897:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077015.123910:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077015.123918:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077015/real 1769077015] req@ffff9b0343dec380 x1855007617336064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077015.123927:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617336064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077015.123935:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077015.123938:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077015.123941:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077015.123945:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077015.123957:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077015.123976:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077015/real 1769077015] req@ffff9b0343ded180 x1855007617336192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077015.123986:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617336192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077015.123993:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077015.123995:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077015.123998:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077015.635387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077016.019425:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077016.019478:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077016.019590:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077016.659354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077017.043357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077017.043358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077017.043464:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077017.683469:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077018.067377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077018.067377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077018.067483:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077018.707323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077019.091373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077019.091383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077019.091538:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077019.731383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077020.115440:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077020.115465:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077020.115545:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077020.243453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077020.243464:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077020.243468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077020.243478:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4009 00000100:00080000:3.0:1769077020.243488:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077020.243496:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077020.243534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617336320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077020.243562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077020.243566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077020.243568:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077020.243571:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4009 00000100:00080000:3.0:1769077020.243574:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077020.243577:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077020.243586:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077020.243598:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617336448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077020.243609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077020.243616:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769077020.243621:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617336320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077020.243636:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077020.243652:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4009 00000100:00080000:3.0:1769077020.243657:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077020.243661:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077020.243667:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077020.243682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617336576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077020.243691:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077020.243696:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077020.243698:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077020.243702:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4009 00000100:00080000:3.0:1769077020.243718:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077020.243723:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077020.243739:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077020.243748:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617336704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077020.243757:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077020.243771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077020.243775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077020.243777:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077020.243781:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4009 00000100:00080000:3.0:1769077020.243785:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077020.243789:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077020.243794:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077020.243806:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617336832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077020.243849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077020/real 1769077020] req@ffff9b0352a18700 x1855007617336320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077020.243883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617336320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077020.243896:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077020.243902:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077020.243906:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617336320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077020.243937:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077020.243970:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077020.243982:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077020/real 1769077020] req@ffff9b0352a1aa00 x1855007617336448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077020.243993:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617336448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077020.244001:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077020.244004:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077020.244007:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077020.244012:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077020.244027:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077020.244034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077020/real 1769077020] req@ffff9b0352a18e00 x1855007617336576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077020.244043:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617336576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077020.244050:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077020.244053:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077020.244055:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077020.244061:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077020.244074:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077020.244080:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077020/real 1769077020] req@ffff9b0352a1a300 x1855007617336704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077020.244089:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617336704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077020.244095:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077020.244098:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077020.244101:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077020.244106:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077020.244117:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077020.244124:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077020/real 1769077020] req@ffff9b0352a19f80 x1855007617336832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077020.244132:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617336832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077020.244138:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077020.244140:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077020.244143:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617336832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077020.755347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077021.139324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077021.139388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077021.139431:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077021.779341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077022.163384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077022.163441:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077022.163441:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077022.803434:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077023.187383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077023.187383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077023.187459:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077023.827405:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077024.211420:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077024.211422:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077024.211463:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077024.851375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077025.235430:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077025.235441:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077025.235517:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077025.363500:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077025.363515:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077025.363523:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077025.363536:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4014 00000100:00080000:2.0:1769077025.363551:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077025.363562:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077025.363633:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617336960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077025.363682:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077025.363686:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077025.363688:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077025.363693:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4014 00000100:00080000:2.0:1769077025.363698:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077025.363702:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077025.363712:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077025.363738:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617337088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077025.363749:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077025.363754:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077025.363756:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077025.363760:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4014 00000100:00080000:2.0:1769077025.363763:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077025.363767:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077025.363773:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077025.363779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617336960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077025.363787:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617337216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077025.363806:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077025.363808:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077025.363810:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077025.363815:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4014 00000100:00080000:2.0:1769077025.363819:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077025.363822:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077025.363827:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077025.363837:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617337344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077025.363845:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077025.363849:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077025.363851:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077025.363855:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4014 00000100:00080000:2.0:1769077025.363858:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077025.363862:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077025.363867:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077025.363882:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617337472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077025.363889:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077025.363956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077025/real 1769077025] req@ffff9b03482c6d80 x1855007617336960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077025.363983:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617336960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077025.363993:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077025.363997:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077025.364000:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617336960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077025.364007:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077025.364025:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077025.364034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077025/real 1769077025] req@ffff9b03482c6680 x1855007617337088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077025.364042:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617337088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077025.364049:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077025.364051:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077025.364053:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077025.364078:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077025.364089:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077025.364095:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077025/real 1769077025] req@ffff9b03482c7100 x1855007617337216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077025.364102:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617337216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077025.364113:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077025.364115:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077025.364117:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077025.364121:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077025.364131:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077025.364137:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077025/real 1769077025] req@ffff9b03482c4000 x1855007617337344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077025.364144:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617337344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077025.364150:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077025.364152:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077025.364154:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077025.364158:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077025.364168:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077025.364174:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077025/real 1769077025] req@ffff9b03482c6a00 x1855007617337472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077025.364180:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617337472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077025.364185:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077025.364188:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077025.364191:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077025.875426:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077026.259530:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077026.259542:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077026.259732:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077026.899342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077027.283468:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077027.283553:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077027.283558:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077027.923403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077028.307353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077028.307876:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077028.307974:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077028.947319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769077029.331566:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077029.331726:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077029.332346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077029.971348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077030.355420:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077030.355498:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077030.355513:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769077030.483366:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077030.483375:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077030.483379:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077030.483388:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4019 00000100:00080000:3.0:1769077030.483397:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077030.483403:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077030.483437:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617337600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077030.483460:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077030.483463:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077030.483464:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077030.483467:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4019 00000100:00080000:3.0:1769077030.483469:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077030.483472:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077030.483480:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077030.483492:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617337728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077030.483497:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617337600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077030.483504:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077030.483507:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077030.483508:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077030.483511:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4019 00000100:00080000:3.0:1769077030.483513:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077030.483516:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077030.483520:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077030.483528:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617337856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077030.483534:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077030.483536:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077030.483537:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077030.483540:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4019 00000100:00080000:3.0:1769077030.483542:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077030.483544:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077030.483548:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077030.483556:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617337984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077030.483563:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077030.483570:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077030.483573:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077030.483574:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077030.483579:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4019 00000100:00080000:3.0:1769077030.483583:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077030.483586:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077030.483588:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077030.483596:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617338112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077030.483606:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077030/real 1769077030] req@ffff9b0352a19f80 x1855007617337600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077030.483621:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617337600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077030.483629:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077030.483633:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077030.483636:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617337600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077030.483655:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077030.483674:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077030.483683:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077030/real 1769077030] req@ffff9b0352a1a300 x1855007617337728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077030.483691:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617337728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077030.483697:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077030.483700:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077030.483702:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077030.483707:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077030.483720:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077030.483726:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077030/real 1769077030] req@ffff9b0352a18e00 x1855007617337856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077030.483736:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617337856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077030.483742:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077030.483744:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077030.483748:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077030.483752:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077030.483765:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077030.483774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077030/real 1769077030] req@ffff9b0352a1aa00 x1855007617337984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077030.483782:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617337984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077030.483787:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077030.483790:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077030.483792:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617337984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077030.483795:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617338112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077030.483804:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077030.483810:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077030/real 1769077030] req@ffff9b0352a18700 x1855007617338112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077030.483817:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617338112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077030.483822:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077030.483824:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077030.483826:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617338112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077030.995328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077031.379546:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077031.379560:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077031.379571:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077032.019334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077032.403327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077032.403327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077032.403404:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077033.043339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077033.427378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077033.427380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077033.427475:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077034.067329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077034.451350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077034.451387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077034.451434:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077035.091422:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077035.475408:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077035.475419:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077035.475498:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077035.603338:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077035.603354:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077035.603360:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077035.603374:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4024 00000100:00080000:3.0:1769077035.603387:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077035.603396:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077035.603449:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617338240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077035.603480:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077035.603484:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077035.603487:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077035.603492:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4024 00000100:00080000:3.0:1769077035.603497:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077035.603502:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077035.603532:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077035.603541:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617338240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077035.603546:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617338368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077035.603556:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077035.603560:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077035.603561:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077035.603565:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4024 00000100:00080000:3.0:1769077035.603568:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077035.603572:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077035.603577:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077035.603586:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617338496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077035.603594:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077035.603608:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077035.603610:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077035.603613:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4024 00000100:00080000:3.0:1769077035.603617:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077035.603621:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769077035.603622:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769077035.603626:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077035.603635:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617338624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077035.603644:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077035.603647:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077035.603649:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077035.603653:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4024 00000100:00080000:3.0:1769077035.603656:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077035.603659:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077035.603664:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077035.603669:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077035/real 1769077035] req@ffff9b0352a1a680 x1855007617338240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077035.603673:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617338752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077035.603690:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617338240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077035.603702:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077035.603707:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077035.603712:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617338240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077035.603750:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617338368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077035.603787:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077035.603798:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077035/real 1769077035] req@ffff9b0352a18a80 x1855007617338368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077035.603807:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617338368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077035.603814:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077035.603816:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077035.603819:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617338368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077035.603823:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617338496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077035.603835:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077035.603841:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077035/real 1769077035] req@ffff9b0352a1b480 x1855007617338496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077035.603849:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617338496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077035.603855:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077035.603857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077035.603859:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617338496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077035.603863:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617338624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077035.603874:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077035.603880:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077035/real 1769077035] req@ffff9b0352a1ad80 x1855007617338624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077035.603888:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617338624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077035.603894:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077035.603896:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077035.603898:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617338624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077035.603901:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617338752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077035.603911:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077035.603916:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077035/real 1769077035] req@ffff9b0352a1b100 x1855007617338752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077035.603923:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617338752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077035.603929:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077035.603931:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077035.603933:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617338752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077036.115335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077036.499350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077036.499412:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077036.499447:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077037.139425:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077037.523344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077037.523345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077037.523425:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077038.163422:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077038.547368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077038.547437:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077038.547477:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077039.187362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077039.571389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077039.571414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077039.571470:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077040.211360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077040.595355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077040.595355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077040.595407:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077040.723402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077040.723418:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077040.723425:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077040.723439:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4029 00000100:00080000:2.0:1769077040.723455:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077040.723475:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077040.723548:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617338880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077040.723585:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077040.723590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077040.723593:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077040.723598:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4029 00000100:00080000:2.0:1769077040.723604:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077040.723609:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077040.723622:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077040.723638:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617339008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077040.723653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077040.723658:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077040.723668:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077040.723674:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4029 00000100:00080000:2.0:1769077040.723678:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077040.723686:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077040.723695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617338880:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769077040.723695:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077040.723716:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617339136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077040.723730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077040.723734:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077040.723737:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077040.723741:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4029 00000100:00080000:2.0:1769077040.723745:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077040.723749:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077040.723755:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077040.723771:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617339264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077040.723791:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077040.723796:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077040.723798:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077040.723802:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4029 00000100:00080000:2.0:1769077040.723806:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077040.723809:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077040.723815:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769077040.723824:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077040.723834:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617339392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077040.723904:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077040/real 1769077040] req@ffff9b03482c4700 x1855007617338880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077040.723935:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617338880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077040.723959:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077040.723965:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077040.723970:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617338880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077040.723981:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077040.724013:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077040.724029:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077040/real 1769077040] req@ffff9b03482c5880 x1855007617339008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077040.724073:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617339008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077040.724093:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077040.724098:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077040.724111:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077040.724147:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077040.724174:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077040.724188:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077040/real 1769077040] req@ffff9b03482c7b80 x1855007617339136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077040.724203:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617339136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077040.724220:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077040.724224:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077040.724228:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077040.724266:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077040.724291:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077040.724304:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077040/real 1769077040] req@ffff9b03482c5f80 x1855007617339264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077040.724318:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617339264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077040.724328:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077040.724331:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077040.724335:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077040.724341:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077040.724358:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077040.724367:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077040/real 1769077040] req@ffff9b03482c5c00 x1855007617339392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077040.724380:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617339392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077040.724389:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077040.724392:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077040.724397:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077041.235418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077041.619324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077041.619328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077041.619481:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077042.259411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077042.643305:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077042.643312:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077042.643421:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077043.283370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077043.667317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077043.667372:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077043.667384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077044.307304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077044.691325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077044.691325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077044.691398:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077045.331352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077045.715320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077045.715355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077045.715416:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077045.843532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077045.843545:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077045.843550:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077045.843561:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4035 00000100:00080000:0.0:1769077045.843572:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077045.843579:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077045.843636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617339520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077045.843670:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077045.843674:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077045.843675:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077045.843679:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4035 00000100:00080000:0.0:1769077045.843681:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077045.843684:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077045.843693:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077045.843703:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617339648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077045.843711:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077045.843713:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077045.843714:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077045.843717:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4035 00000100:00080000:0.0:1769077045.843720:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077045.843725:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617339520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077045.843726:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077045.843731:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077045.843740:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617339776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077045.843746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077045.843748:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077045.843750:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077045.843753:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4035 00000100:00080000:0.0:1769077045.843755:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077045.843758:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077045.843761:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077045.843768:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617339904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077045.843774:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077045.843789:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077045.843790:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077045.843793:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4035 00000100:00080000:0.0:1769077045.843817:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077045.843820:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077045.843823:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077045.843867:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617340032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077045.843911:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077045.844031:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077045.844049:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077045.844066:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077046.355365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077046.739360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077046.739366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077046.739449:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077047.379333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077047.763310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077047.763316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077047.763377:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077048.403321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077048.787361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077048.787361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077048.787423:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769077048.915456:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769077048.915495:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077048.915511:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077048.915550:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077048.915560:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077048.915568:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077048.915577:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769077048.915607:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077045/real 1769077048] req@ffff9b0365a39500 x1855007617339776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077048.915632:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617339776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077048.915646:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077048.915650:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077048.915653:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077048.915660:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077045/real 1769077048] req@ffff9b0365a39180 x1855007617339904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077048.915672:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617339904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077048.915680:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077048.915683:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077048.915686:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077048.915693:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077045/real 1769077048] req@ffff9b0365a3ad80 x1855007617340032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077048.915703:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617340032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077048.915712:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077048.915716:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077048.915720:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077048.915728:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077045/real 1769077048] req@ffff9b0365a3bb80 x1855007617339520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077048.915749:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617339520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077048.915758:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077048.915761:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077048.915765:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617339520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077048.915772:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077045/real 1769077048] req@ffff9b0365a3b100 x1855007617339648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077048.915786:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617339648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077048.915795:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077048.915799:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077048.915803:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617339648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077049.427315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077049.811382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077049.811383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077049.811487:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077050.451514:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077050.835375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077050.835442:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077050.835450:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077050.963310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077050.963321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077050.963324:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077050.963334:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4040 00000100:00080000:0.0:1769077050.963346:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077050.963353:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077050.963388:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617340160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077050.963416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077050.963420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077050.963427:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077050.963432:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4040 00000100:00080000:0.0:1769077050.963436:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077050.963440:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077050.963449:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077050.963462:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617340288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077050.963471:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617340160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077050.963479:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077050.963482:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077050.963484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077050.963488:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4040 00000100:00080000:0.0:1769077050.963491:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077050.963494:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077050.963499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077050.963508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617340416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077050.963516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077050.963519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077050.963520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077050.963523:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4040 00000100:00080000:0.0:1769077050.963526:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077050.963530:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769077050.963532:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769077050.963534:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077050.963542:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617340544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077050.963551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077050.963554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077050.963555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077050.963558:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4040 00000100:00080000:0.0:1769077050.963562:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077050.963566:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077050.963569:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077050/real 1769077050] req@ffff9b0365a3b100 x1855007617340160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077050.963570:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077050.963578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617340672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077050.963586:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617340160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077050.963597:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077050.963602:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077050.963606:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617340160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077050.963628:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077050.963653:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077050.963667:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077050/real 1769077050] req@ffff9b0365a3bb80 x1855007617340288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077050.963680:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617340288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077050.963689:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077050.963693:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077050.963696:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077050.963702:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077050.963728:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077050.963740:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077050/real 1769077050] req@ffff9b0365a3ad80 x1855007617340416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077050.963753:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617340416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077050.963762:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077050.963765:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077050.963769:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077050.963776:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077050.963796:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077050.963808:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077050/real 1769077050] req@ffff9b0365a39180 x1855007617340544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077050.963822:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617340544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077050.963831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077050.963835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077050.963839:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077050.963846:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077050.963901:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077050.963914:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077050/real 1769077050] req@ffff9b0365a39500 x1855007617340672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077050.963927:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617340672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077050.963936:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077050.963939:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077050.963942:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077051.475318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077051.859358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077051.859358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077051.859530:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077052.499303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077052.883342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077052.883342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077052.883404:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077053.523312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077053.907337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077053.907372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077053.907374:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077054.547314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077054.931341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077054.931341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077054.931438:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077055.571292:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077055.955363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077055.955392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077055.955509:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077056.083402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077056.083413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077056.083417:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077056.083427:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4045 00000100:00080000:1.0:1769077056.083437:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077056.083444:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077056.083480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617340800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077056.083524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077056.083527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077056.083529:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077056.083533:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4045 00000100:00080000:1.0:1769077056.083536:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077056.083539:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077056.083548:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077056.083560:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617340928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077056.083570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077056.083573:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617340800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077056.083574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077056.083578:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077056.083587:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4045 00000100:00080000:1.0:1769077056.083590:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077056.083593:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077056.083597:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077056.083609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617341056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077056.083618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077056.083621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077056.083623:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077056.083642:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4045 00000100:00080000:1.0:1769077056.083645:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077056.083647:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077056.083651:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077056.083658:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617341184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077056.083664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077056.083666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077056.083667:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077056.083670:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4045 00000800:00000100:2.0:1769077056.083675:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077056.083676:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077056.083682:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077056.083686:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077056.083693:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617341312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077056.083729:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077056/real 1769077056] req@ffff9b0343dec000 x1855007617340800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077056.083744:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617340800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077056.083752:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077056.083755:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077056.083758:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617340800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077056.083805:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077056.083841:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077056.083853:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077056/real 1769077056] req@ffff9b0343dec700 x1855007617340928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077056.083863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617340928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077056.083871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077056.083874:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077056.083876:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617340928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077056.083882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077056.083900:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077056.083913:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077056/real 1769077056] req@ffff9b0343ded180 x1855007617341056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077056.083929:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617341056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077056.083941:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077056.083945:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077056.083949:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077056.083958:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077056.083984:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077056.083995:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077056/real 1769077056] req@ffff9b0343dec380 x1855007617341184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077056.084005:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617341184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077056.084011:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077056.084014:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077056.084017:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077056.084021:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077056.084034:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077056.084041:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077056/real 1769077056] req@ffff9b0343dee300 x1855007617341312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077056.084051:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617341312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077056.084057:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077056.084059:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077056.084062:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077056.595369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077056.979388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077056.979396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077056.979510:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077057.619423:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077058.003328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077058.003382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077058.003423:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077058.643372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769077059.027391:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077059.027392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077059.027392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077059.667354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077060.051418:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077060.051425:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077060.051482:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077060.691363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077061.075329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077061.075331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077061.075402:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077061.203360:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077061.203373:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077061.203380:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077061.203393:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4050 00000100:00080000:1.0:1769077061.203416:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077061.203426:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077061.203468:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617341440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077061.203519:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077061.203523:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077061.203526:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077061.203532:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4050 00000100:00080000:1.0:1769077061.203537:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077061.203542:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077061.203553:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077061.203557:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617341440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077061.203589:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617341568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077061.203606:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077061.203611:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077061.203614:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077061.203619:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4050 00000100:00080000:1.0:1769077061.203626:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077061.203632:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077061.203639:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077061.203644:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077061.203661:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617341696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077061.203686:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077061.203686:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077061/real 1769077061] req@ffff9b0343def100 x1855007617341440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077061.203736:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617341440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077061.203742:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077061.203749:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077061.203754:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077061.203761:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4050 00000100:00080000:2.0:1769077061.203762:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077061.203782:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617341440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077061.203784:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077061.203794:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077061.203801:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077061.203807:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077061.203836:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077061.203838:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617341824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077061.203850:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077061/real 1769077061] req@ffff9b0343dedf80 x1855007617341568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077061.203859:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077061.203867:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617341568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077061.203868:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077061.203874:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077061.203886:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077061.203889:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4050 00000100:00080000:2.0:1769077061.203893:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077061.203899:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341568:192.168.204.160@tcp:38:kworker.0 00000100:00080000:1.0:1769077061.203901:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077061.203907:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341696:192.168.204.160@tcp:38:kworker.0 00000100:00080000:1.0:1769077061.203914:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077061.203923:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769077061.203930:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077061.203943:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617341952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077061.203944:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077061/real 1769077061] req@ffff9b0343ded880 x1855007617341696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077061.203986:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617341696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077061.203994:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077061.203997:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077061.204000:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077061.204014:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077061.204027:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077061.204034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077061/real 1769077061] req@ffff9b0343def800 x1855007617341824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077061.204043:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617341824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077061.204055:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077061.204057:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077061.204060:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077061.204064:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077061.204075:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077061.204081:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077061/real 1769077061] req@ffff9b0343deea00 x1855007617341952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077061.204090:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617341952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077061.204106:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077061.204108:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077061.204110:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617341952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077061.715420:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077062.099338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077062.099338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077062.099425:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077062.739403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077063.123315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077063.123316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077063.123377:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077063.763316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077064.147363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077064.147363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077064.147389:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077064.787337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077065.171352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077065.171385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077065.171432:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077065.811345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077066.195349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077066.195349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077066.195411:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077066.323311:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077066.323320:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077066.323324:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077066.323334:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4055 00000100:00080000:1.0:1769077066.323343:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077066.323350:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077066.323379:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617342080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077066.323404:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077066.323407:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077066.323409:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077066.323412:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4055 00000100:00080000:1.0:1769077066.323414:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077066.323417:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077066.323425:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077066.323439:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617342208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077066.323439:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617342080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077066.323453:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077066.323456:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077066.323457:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077066.323461:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4055 00000100:00080000:1.0:1769077066.323464:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077066.323467:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077066.323471:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077066.323482:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617342336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077066.323490:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077066.323493:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077066.323495:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077066.323497:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4055 00000100:00080000:1.0:1769077066.323500:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077066.323505:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077066.323505:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769077066.323513:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077066.323521:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617342464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077066.323528:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077066.323531:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077066.323533:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077066.323536:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4055 00000100:00080000:1.0:1769077066.323539:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077066.323541:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077066.323545:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077066.323553:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617342592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077066.323630:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077066/real 1769077066] req@ffff9b0343deca80 x1855007617342080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077066.323649:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617342080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077066.323763:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077066.323770:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077066.323775:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617342080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077066.323800:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077066.323865:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077066.323879:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077066/real 1769077066] req@ffff9b0343deed80 x1855007617342208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077066.323894:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617342208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077066.323904:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077066.323907:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077066.323911:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077066.323917:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077066.323933:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077066.323941:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077066/real 1769077066] req@ffff9b0343dee680 x1855007617342336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077066.323949:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617342336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077066.323955:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077066.323957:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077066.323959:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077066.323964:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077066.323974:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077066.323980:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077066/real 1769077066] req@ffff9b0343defb80 x1855007617342464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077066.323987:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617342464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077066.323992:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077066.323994:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077066.323996:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077066.323999:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077066.324009:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077066.324014:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077066/real 1769077066] req@ffff9b0343dece00 x1855007617342592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077066.324021:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617342592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077066.324026:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077066.324028:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077066.324030:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077066.835374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077067.219328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077067.219328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077067.219444:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077067.859382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077068.243405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077068.243421:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077068.243499:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077068.883329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077069.267300:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077069.267327:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077069.267354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077069.907326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077070.291382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077070.291382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077070.291527:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077070.931382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077071.315312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077071.315312:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077071.315404:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077071.443372:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077071.443384:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077071.443390:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077071.443401:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4060 00000100:00080000:2.0:1769077071.443412:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077071.443419:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077071.443463:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617342720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077071.443488:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077071.443492:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077071.443493:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077071.443497:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4060 00000100:00080000:2.0:1769077071.443501:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077071.443506:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077071.443518:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077071.443543:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617342720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077071.443549:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617342848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077071.443564:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077071.443568:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077071.443571:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077071.443575:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4060 00000100:00080000:2.0:1769077071.443580:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077071.443585:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077071.443590:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077071.443600:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617342976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077071.443612:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077071.443615:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077071.443616:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769077071.443621:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077071.443622:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4060 00000100:00080000:2.0:1769077071.443628:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077071.443630:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077071.443634:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077071.443643:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617343104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077071.443650:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077071.443653:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077071.443654:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077071.443657:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4060 00000100:00080000:2.0:1769077071.443659:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077071.443662:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077071.443665:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077071.443671:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077071/real 1769077071] req@ffff9b03482c4e00 x1855007617342720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077071.443676:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617343232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077071.443710:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617342720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077071.443720:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077071.443724:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077071.443728:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617342720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077071.443751:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077071.443767:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077071.443775:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077071/real 1769077071] req@ffff9b03482c5c00 x1855007617342848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077071.443784:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617342848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077071.443790:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077071.443792:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077071.443794:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077071.443799:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077071.443809:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077071.443815:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077071/real 1769077071] req@ffff9b03482c5f80 x1855007617342976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077071.443823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617342976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077071.443828:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077071.443830:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077071.443831:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617342976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077071.443835:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077071.443845:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077071.443851:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077071/real 1769077071] req@ffff9b03482c7b80 x1855007617343104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077071.443858:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617343104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077071.443864:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077071.443865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077071.443867:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077071.443879:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077071.443890:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077071.443897:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077071/real 1769077071] req@ffff9b03482c5880 x1855007617343232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077071.443909:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617343232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077071.443918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077071.443921:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077071.443925:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077071.955361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077072.339347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077072.339415:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077072.339466:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077072.979388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077073.363381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077073.363381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077073.363442:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077074.003341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077074.387371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077074.387372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077074.387375:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077075.027336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077075.411327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077075.411359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077075.411423:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077076.051380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077076.435369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077076.435467:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077076.435496:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077076.563365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077076.563390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077076.563397:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077076.563410:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4065 00000100:00080000:0.0:1769077076.563424:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077076.563474:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077076.563528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617343360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077076.563570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077076.563574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077076.563577:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077076.563582:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4065 00000100:00080000:0.0:1769077076.563587:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077076.563592:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077076.563605:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077076.563622:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617343488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077076.563634:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617343360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077076.563641:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077076.563650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077076.563652:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077076.563658:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4065 00000100:00080000:0.0:1769077076.563662:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077076.563668:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077076.563674:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077076.563699:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617343616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077076.563710:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077076.563714:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077076.563717:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077076.563724:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4065 00000800:00000100:3.0:1769077076.563724:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077076.563736:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077076.563744:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077076.563751:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077076.563771:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617343744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077076.563782:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077076.563787:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077076/real 1769077076] req@ffff9b0365a3aa00 x1855007617343360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077076.563789:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077076.563794:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077076.563809:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4065 00000100:00080000:0.0:1769077076.563818:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077076.563825:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077076.563832:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077076.563838:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617343360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077076.563853:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617343872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077076.563854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077076.563869:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077076.563880:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617343360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077076.563913:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077076.563940:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077076.563956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077076/real 1769077076] req@ffff9b0365a3b480 x1855007617343488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077076.563970:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617343488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077076.563981:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077076.563985:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077076.563988:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077076.563995:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077076.564016:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077076.564028:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077076/real 1769077076] req@ffff9b0365a38e00 x1855007617343616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077076.564041:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617343616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077076.564050:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077076.564054:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077076.564057:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077076.564064:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077076.564085:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077076.564097:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077076/real 1769077076] req@ffff9b0365a39c00 x1855007617343744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077076.564131:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617343744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077076.564141:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077076.564145:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077076.564149:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077076.564156:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077076.564180:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077076.564194:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077076/real 1769077076] req@ffff9b0365a38a80 x1855007617343872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077076.564209:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617343872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077076.564218:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077076.564221:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077076.564225:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617343872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077077.075349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077077.459385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077077.459461:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077077.459462:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077078.099330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077078.483338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077078.483338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077078.483447:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077079.123361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077079.507343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077079.507383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077079.507401:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077080.147481:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077080.531395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077080.531396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077080.531415:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077081.171352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077081.555382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077081.555382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077081.555453:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077081.683312:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077081.683324:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077081.683329:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077081.683340:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4070 00000100:00080000:0.0:1769077081.683361:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077081.683370:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077081.683416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617344000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077081.683442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077081.683445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077081.683447:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077081.683450:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4070 00000100:00080000:0.0:1769077081.683454:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077081.683457:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077081.683467:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077081.683480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617344128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077081.683480:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617344000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077081.683501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077081.683504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077081.683505:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077081.683509:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4070 00000100:00080000:0.0:1769077081.683511:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077081.683514:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077081.683517:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077081.683527:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617344256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077081.683533:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077081.683536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077081.683537:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077081.683541:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4070 00000100:00080000:0.0:1769077081.683554:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077081.683560:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077081.683564:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769077081.683571:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077081.683596:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617344384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077081.683609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077081.683613:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077081.683618:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077081.683618:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077081/real 1769077081] req@ffff9b0365a38a80 x1855007617344000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077081.683626:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4070 00000100:00080000:0.0:1769077081.683634:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077081.683640:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077081.683645:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077081.683654:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617344000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077081.683664:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617344512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077081.683672:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077081.683680:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077081.683687:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617344000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077081.683715:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077081.683736:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077081.683750:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077081/real 1769077081] req@ffff9b0365a39c00 x1855007617344128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077081.683764:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617344128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077081.683774:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077081.683778:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077081.683782:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077081.683790:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077081.683808:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077081.683816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077081/real 1769077081] req@ffff9b0365a38e00 x1855007617344256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077081.683825:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617344256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077081.683831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077081.683833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077081.683835:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077081.683839:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077081.683851:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077081.683857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077081/real 1769077081] req@ffff9b0365a3b480 x1855007617344384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077081.683865:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617344384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077081.683870:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077081.683872:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077081.683874:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077081.683877:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077081.683886:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077081.683891:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077081/real 1769077081] req@ffff9b0365a3aa00 x1855007617344512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077081.683899:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617344512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077081.683909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077081.683911:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077081.683913:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077082.195382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077082.579345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077082.579420:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077082.579588:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077083.219351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077083.603367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077083.603419:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077083.603449:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077084.243376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077084.627354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077084.627354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077084.627377:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077085.267306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077085.651294:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077085.651305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077085.651398:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077086.291365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077086.675314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077086.675383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077086.675396:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077086.803397:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077086.803407:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077086.803411:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077086.803420:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4075 00000100:00080000:1.0:1769077086.803429:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077086.803434:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077086.803468:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617344640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077086.803491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077086.803494:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077086.803495:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077086.803499:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4075 00000100:00080000:1.0:1769077086.803502:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077086.803505:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077086.803514:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077086.803526:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617344768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077086.803526:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617344640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077086.803547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077086.803550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077086.803551:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077086.803555:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4075 00000100:00080000:1.0:1769077086.803558:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077086.803561:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077086.803564:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077086.803574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617344896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077086.803581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077086.803584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077086.803585:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077086.803587:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4075 00000100:00080000:1.0:1769077086.803590:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077086.803594:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077086.803594:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769077086.803600:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077086.803608:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617345024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077086.803615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077086.803617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077086.803618:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077086.803621:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4075 00000100:00080000:1.0:1769077086.803624:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077086.803627:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077086.803631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077086.803635:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077086/real 1769077086] req@ffff9b0343def480 x1855007617344640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077086.803641:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617345152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077086.803667:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617344640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077086.803677:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077086.803681:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077086.803684:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617344640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077086.803701:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077086.803718:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077086.803726:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077086/real 1769077086] req@ffff9b0343ded500 x1855007617344768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077086.803734:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617344768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077086.803740:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077086.803742:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077086.803744:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077086.803748:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077086.803758:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077086.803763:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077086/real 1769077086] req@ffff9b0343dece00 x1855007617344896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077086.803770:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617344896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077086.803775:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077086.803777:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077086.803779:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617344896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077086.803782:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077086.803792:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077086.803797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077086/real 1769077086] req@ffff9b0343defb80 x1855007617345024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077086.803809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617345024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077086.803814:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077086.803815:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077086.803817:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077086.803821:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077086.803829:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077086.803834:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077086/real 1769077086] req@ffff9b0343dee680 x1855007617345152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077086.803841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617345152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077086.803848:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077086.803851:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077086.803855:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077087.315348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077087.699318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077087.699355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077087.699401:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077088.339353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077088.723313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077088.723329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077088.723381:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077089.363361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077089.747326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077089.747326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077089.747346:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077090.387376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077090.771357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077090.771358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077090.771423:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077091.411379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077091.795373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077091.795378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077091.795448:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077091.923353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077091.923364:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077091.923368:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077091.923379:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4081 00000100:00080000:0.0:1769077091.923390:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077091.923396:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077091.923442:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617345280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077091.923470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077091.923473:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077091.923475:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077091.923479:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4081 00000100:00080000:0.0:1769077091.923482:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077091.923485:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077091.923494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077091.923506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617345408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077091.923506:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617345280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077091.923527:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077091.923530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077091.923532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077091.923535:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4081 00000100:00080000:0.0:1769077091.923538:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077091.923540:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077091.923543:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077091.923551:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617345536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077091.923563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077091.923566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077091.923568:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077091.923571:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4081 00000100:00080000:0.0:1769077091.923574:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077091.923576:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077091.923579:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077091.923587:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617345664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077091.923594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077091.923596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077091.923598:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077091.923600:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4081 00000800:00000100:3.0:1769077091.923606:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077091.923607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077091.923613:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077091.923616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077091.923625:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617345792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077091.923657:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077091/real 1769077091] req@ffff9b0365a39500 x1855007617345280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077091.923675:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617345280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077091.923685:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077091.923688:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077091.923692:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617345280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077091.923716:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077091.923733:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077091.923741:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077091/real 1769077091] req@ffff9b0365a39180 x1855007617345408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077091.923750:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617345408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077091.923756:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077091.923758:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077091.923761:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077091.923764:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077091.923774:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077091.923780:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077091/real 1769077091] req@ffff9b0365a3ad80 x1855007617345536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077091.923788:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617345536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077091.923794:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077091.923796:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077091.923798:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077091.923801:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077091.923811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077091.923817:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077091/real 1769077091] req@ffff9b0365a3bb80 x1855007617345664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077091.923823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617345664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077091.923829:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077091.923831:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077091.923833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077091.923836:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077091.923845:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077091.923850:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077091/real 1769077091] req@ffff9b0365a3b100 x1855007617345792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077091.923858:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617345792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077091.923873:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077091.923876:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077091.923880:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617345792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077092.435317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769077092.819718:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077092.819784:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077092.820016:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077093.459334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077093.843373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077093.843395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077093.843497:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077094.483361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077094.867438:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077094.867480:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077094.867544:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077095.507384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077095.891383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077095.891574:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077095.891644:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077096.531474:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077096.915309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077096.915449:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077096.915541:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769077097.043421:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077097.043430:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077097.043433:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077097.043442:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4086 00000100:00080000:2.0:1769077097.043452:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077097.043458:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077097.043492:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617345920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077097.043516:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077097.043519:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077097.043520:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077097.043524:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4086 00000100:00080000:2.0:1769077097.043526:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077097.043529:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077097.043536:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077097.043545:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617346048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077097.043554:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077097.043557:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077097.043558:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077097.043565:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4086 00000100:00100000:3.0:1769077097.043567:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617345920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077097.043568:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077097.043580:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077097.043585:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077097.043593:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617346176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077097.043600:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077097.043602:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077097.043605:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077097.043608:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4086 00000100:00080000:2.0:1769077097.043611:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077097.043613:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077097.043617:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077097.043627:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617346304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077097.043635:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077097.043637:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077097.043638:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077097.043641:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4086 00000100:00080000:2.0:1769077097.043646:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077097.043649:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077097.043653:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077097.043661:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617346432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077097.043666:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077097.043718:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff9b03482c4700 x1855007617345920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077097.043735:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617345920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077097.043745:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077097.043749:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077097.043752:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617345920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077097.043778:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077097.043796:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077097.043807:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff9b03482c6a00 x1855007617346048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077097.043822:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617346048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077097.043832:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077097.043835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077097.043839:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077097.043858:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077097.043879:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077097.043888:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff9b03482c4000 x1855007617346176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077097.043897:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617346176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077097.043903:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077097.043905:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077097.043907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077097.043912:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077097.043923:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077097.043929:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff9b03482c7100 x1855007617346304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077097.043936:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617346304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077097.043946:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077097.043948:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077097.043950:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077097.043954:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077097.043964:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077097.043970:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077097/real 1769077097] req@ffff9b03482c6680 x1855007617346432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077097.043977:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617346432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077097.043982:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077097.043984:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077097.043986:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077097.555474:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077097.939467:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077097.939632:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077097.939635:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077098.579339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077098.963414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077098.963414:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077098.963521:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077099.603395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077099.987352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077099.987352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077099.987430:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077100.627337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077101.011347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077101.011380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077101.011463:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077101.651379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077102.035405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077102.035405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077102.035511:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077102.163421:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077102.163438:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077102.163445:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077102.163465:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4091 00000100:00080000:2.0:1769077102.163481:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077102.163494:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077102.163552:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617346560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077102.163633:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077102.163643:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077102.163648:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077102.163660:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4091 00000100:00080000:2.0:1769077102.163670:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077102.163680:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077102.163697:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077102.163727:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617346688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077102.163752:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077102.163760:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077102.163764:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077102.163775:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4091 00000100:00080000:2.0:1769077102.163782:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077102.163790:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077102.163801:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077102.163823:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617346816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077102.163859:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077102.163867:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077102.163871:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077102.163878:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4091 00000100:00080000:2.0:1769077102.163884:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077102.163890:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077102.163899:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077102.163917:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617346944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077102.163934:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077102.163940:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077102.163943:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077102.163950:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4091 00000100:00080000:2.0:1769077102.163957:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077102.163965:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077102.163973:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077102.163994:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617347072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077102.164110:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617346560:192.168.204.160@tcp:250:kworker.0 00000800:00000100:0.0:1769077102.164195:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077102.164282:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077102/real 1769077102] req@ffff9b03482c6d80 x1855007617346560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077102.164314:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617346560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077102.164328:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077102.164334:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077102.164339:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617346560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077102.164348:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077102.164377:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077102.164392:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077102/real 1769077102] req@ffff9b03482c6300 x1855007617346688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077102.164407:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617346688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077102.164426:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077102.164430:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077102.164434:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077102.164441:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077102.164459:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077102.164469:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077102/real 1769077102] req@ffff9b03482c5500 x1855007617346816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077102.164483:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617346816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077102.164493:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077102.164496:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077102.164499:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077102.164506:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077102.164524:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077102.164534:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077102/real 1769077102] req@ffff9b03482c4a80 x1855007617346944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077102.164548:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617346944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077102.164557:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077102.164560:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077102.164565:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617346944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077102.164572:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077102.164589:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077102.164598:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077102/real 1769077102] req@ffff9b03482c5180 x1855007617347072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077102.164612:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617347072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077102.164622:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077102.164625:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077102.164629:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077102.675369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077103.059348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077103.059381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077103.059437:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077103.699307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077104.083369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077104.083369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077104.083438:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077104.723317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077105.107342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077105.107344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077105.107413:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077105.747323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077106.131324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077106.131360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077106.131394:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077106.771335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077107.155340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077107.155387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077107.155456:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077107.283383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077107.283392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077107.283396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077107.283405:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4096 00000100:00080000:0.0:1769077107.283415:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077107.283421:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077107.283453:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617347200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077107.283478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077107.283481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077107.283483:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077107.283486:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4096 00000100:00080000:0.0:1769077107.283489:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077107.283494:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077107.283501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077107.283511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617347328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077107.283521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077107.283526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1769077107.283529:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617347200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077107.283530:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077107.283544:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4096 00000100:00080000:0.0:1769077107.283546:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077107.283549:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077107.283552:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077107.283563:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617347456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077107.283570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077107.283572:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077107.283574:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077107.283577:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4096 00000100:00080000:0.0:1769077107.283579:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077107.283582:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077107.283586:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077107.283593:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617347584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077107.283599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077107.283602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077107.283605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077107.283608:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4096 00000100:00080000:0.0:1769077107.283611:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077107.283613:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077107.283616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077107.283620:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077107.283625:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617347712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077107.283680:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077107/real 1769077107] req@ffff9b0365a3b100 x1855007617347200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077107.283700:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617347200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077107.283709:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077107.283713:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077107.283717:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617347200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077107.283743:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077107.283761:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077107.283769:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077107/real 1769077107] req@ffff9b0365a3bb80 x1855007617347328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077107.283782:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617347328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077107.283792:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077107.283796:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077107.283808:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077107.283816:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077107.283841:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077107.283854:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077107/real 1769077107] req@ffff9b0365a3ad80 x1855007617347456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077107.283867:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617347456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077107.283876:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077107.283880:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077107.283884:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077107.283891:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077107.283955:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077107.283968:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077107/real 1769077107] req@ffff9b0365a39180 x1855007617347584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077107.283981:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617347584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077107.283990:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077107.283994:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077107.283997:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077107.284005:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077107.284024:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077107.284035:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077107/real 1769077107] req@ffff9b0365a39500 x1855007617347712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077107.284048:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617347712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077107.284058:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077107.284061:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077107.284065:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077107.795356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077108.179307:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077108.179400:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077108.179447:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077108.819335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077109.203333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077109.203333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077109.203496:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077109.843429:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077110.227351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077110.227352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077110.227453:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077110.867440:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077111.251331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077111.251331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077111.251429:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077111.891355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077112.275324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077112.275324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077112.275453:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077112.403369:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077112.403381:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077112.403387:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077112.403398:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4101 00000100:00080000:3.0:1769077112.403411:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077112.403419:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077112.403465:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617347840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077112.403495:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077112.403499:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077112.403500:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077112.403504:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4101 00000100:00080000:3.0:1769077112.403506:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077112.403509:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077112.403519:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077112.403530:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617347968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077112.403540:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617347840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077112.403545:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077112.403555:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077112.403557:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077112.403562:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4101 00000100:00080000:3.0:1769077112.403567:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077112.403573:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077112.403581:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077112.403597:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617348096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077112.403608:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077112.403612:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077112.403616:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077112.403621:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4101 00000100:00080000:3.0:1769077112.403627:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077112.403633:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077112.403639:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077112.403649:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617348224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077112.403660:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347968:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769077112.403664:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077112.403672:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077112.403676:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077112.403681:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4101 00000100:00080000:3.0:1769077112.403689:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077112.403693:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077112.403699:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077112.403705:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077112.403717:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617348352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077112.403740:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077112.403765:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077112.915308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077113.299321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077113.299371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077113.299476:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077113.939311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077114.323301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077114.323348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077114.323406:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077114.963319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077115.347371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077115.347435:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077115.347457:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769077115.475473:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769077115.475512:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077115.475527:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077115.475576:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077115.475586:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077115.475597:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077115.475610:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769077115.475620:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077112/real 1769077115] req@ffff9b0352a18000 x1855007617348224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077115.475643:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617348224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077115.475654:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077115.475659:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077115.475664:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077115.475675:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077112/real 1769077115] req@ffff9b0352a19c00 x1855007617348352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077115.475688:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617348352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077115.475696:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077115.475699:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077115.475702:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077115.475710:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077112/real 1769077115] req@ffff9b0352a1b800 x1855007617347968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077115.475723:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617347968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077115.475730:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077115.475734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077115.475737:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617347968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077115.475743:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077112/real 1769077115] req@ffff9b0352a19500 x1855007617348096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077115.475755:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617348096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077115.475763:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077115.475767:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077115.475770:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077115.475777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077112/real 1769077115] req@ffff9b0352a18380 x1855007617347840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077115.475812:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617347840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077115.475820:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077115.475823:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077115.475827:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617347840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077115.987309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077116.371339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077116.371339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077116.371398:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077117.011316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077117.395373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077117.395373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077117.395435:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077117.523390:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077117.523403:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077117.523408:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077117.523420:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4106 00000100:00080000:3.0:1769077117.523431:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077117.523438:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077117.523485:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617348480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077117.523525:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077117.523529:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077117.523530:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077117.523533:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4106 00000100:00080000:3.0:1769077117.523535:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077117.523539:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077117.523548:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077117.523559:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617348608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077117.523566:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077117.523570:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077117.523571:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077117.523573:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4106 00000100:00080000:3.0:1769077117.523575:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077117.523578:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077117.523582:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077117.523584:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617348480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077117.523591:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617348736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077117.523600:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077117.523602:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077117.523604:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077117.523607:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4106 00000100:00080000:3.0:1769077117.523609:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077117.523612:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077117.523615:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077117.523623:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617348864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077117.523630:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077117.523634:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077117.523635:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077117.523638:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4106 00000100:00080000:3.0:1769077117.523641:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077117.523643:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077117.523645:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077117.523652:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617348992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077117.523672:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077117.523729:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077117/real 1769077117] req@ffff9b0352a18380 x1855007617348480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077117.523789:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617348480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077117.523803:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077117.523809:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077117.523814:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617348480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077117.523821:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077117.523844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077117.523859:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077117/real 1769077117] req@ffff9b0352a19500 x1855007617348608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077117.523874:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617348608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077117.523884:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077117.523888:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077117.523892:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077117.523924:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077117.523947:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077117.523959:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077117/real 1769077117] req@ffff9b0352a1b800 x1855007617348736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077117.523972:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617348736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077117.523980:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077117.523983:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077117.523986:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077117.523992:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077117.524012:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077117.524023:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077117/real 1769077117] req@ffff9b0352a19c00 x1855007617348864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077117.524034:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617348864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077117.524042:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077117.524045:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077117.524050:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077117.524057:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077117.524076:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077117.524088:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077117/real 1769077117] req@ffff9b0352a18000 x1855007617348992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077117.524117:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617348992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077117.524126:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077117.524128:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077117.524131:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617348992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077118.035295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077118.419400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077118.419400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077118.419518:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077119.059311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077119.443331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077119.443380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077119.443434:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077120.083291:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077120.467337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077120.467337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077120.467406:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077121.107290:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077121.491350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077121.491380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077121.491530:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077122.131293:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077122.515377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077122.515435:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077122.515451:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077122.643325:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077122.643337:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077122.643342:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077122.643353:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4111 00000100:00080000:3.0:1769077122.643364:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077122.643372:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077122.643424:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617349120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077122.643460:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077122.643466:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077122.643468:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077122.643472:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4111 00000100:00080000:3.0:1769077122.643475:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077122.643479:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077122.643488:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077122.643500:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617349248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077122.643509:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077122.643512:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077122.643514:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077122.643517:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4111 00000100:00080000:3.0:1769077122.643519:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077122.643522:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077122.643527:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077122.643533:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617349376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077122.643545:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617349120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077122.643547:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077122.643563:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077122.643565:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077122.643569:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4111 00000100:00080000:3.0:1769077122.643572:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077122.643574:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077122.643580:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077122.643588:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617349504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077122.643594:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077122.643596:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077122.643598:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077122.643601:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4111 00000100:00080000:3.0:1769077122.643603:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077122.643606:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077122.643608:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077122.643618:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617349632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077122.643641:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077122.643698:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077122/real 1769077122] req@ffff9b0352a1b100 x1855007617349120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077122.643715:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617349120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077122.643724:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077122.643727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077122.643730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617349120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077122.643736:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617349248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077122.643754:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077122.643763:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077122/real 1769077122] req@ffff9b0352a1ad80 x1855007617349248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077122.643772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617349248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077122.643778:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077122.643780:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077122.643782:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617349248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077122.643803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617349376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077122.643814:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077122.643820:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077122/real 1769077122] req@ffff9b0352a1b480 x1855007617349376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077122.643828:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617349376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077122.643833:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077122.643835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077122.643837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617349376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077122.643841:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617349504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077122.643851:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077122.643857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077122/real 1769077122] req@ffff9b0352a18a80 x1855007617349504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077122.643864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617349504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077122.643869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077122.643872:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077122.643874:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617349504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077122.643877:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617349632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077122.643886:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077122.643891:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077122/real 1769077122] req@ffff9b0352a1a680 x1855007617349632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077122.643898:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617349632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077122.643903:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077122.643905:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077122.643907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617349632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077123.155340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077123.539358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077123.539383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077123.539437:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077124.179325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077124.563337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077124.563337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077124.563433:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077125.203321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077125.587302:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077125.587302:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077125.587410:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077126.227384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077126.611387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077126.611402:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077126.611481:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077127.251344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077127.635310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077127.635311:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077127.635411:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077127.763350:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077127.763359:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077127.763363:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077127.763372:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4116 00000100:00080000:3.0:1769077127.763381:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077127.763388:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077127.763420:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617349760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077127.763443:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077127.763446:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077127.763447:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077127.763450:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4116 00000100:00080000:3.0:1769077127.763453:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077127.763456:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077127.763463:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077127.763472:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617349888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077127.763476:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617349760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077127.763486:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077127.763490:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077127.763491:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077127.763494:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4116 00000100:00080000:3.0:1769077127.763497:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077127.763500:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077127.763503:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077127.763510:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617350016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077127.763516:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077127.763519:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077127.763521:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077127.763523:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4116 00000100:00080000:3.0:1769077127.763526:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077127.763528:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077127.763532:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077127.763539:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617350144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077127.763543:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077127.763552:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077127.763555:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077127.763556:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077127.763560:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4116 00000100:00080000:3.0:1769077127.763563:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077127.763565:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077127.763568:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077127.763595:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617350272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077127.763612:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077127/real 1769077127] req@ffff9b0352a18700 x1855007617349760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077127.763634:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617349760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077127.763645:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077127.763649:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077127.763654:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617349760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077127.763678:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617349888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077127.763701:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077127.763715:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077127/real 1769077127] req@ffff9b0352a1aa00 x1855007617349888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077127.763724:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617349888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077127.763730:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077127.763732:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077127.763735:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617349888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077127.763739:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077127.763753:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077127.763759:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077127/real 1769077127] req@ffff9b0352a18e00 x1855007617350016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077127.763766:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617350016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077127.763771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077127.763772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077127.763774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077127.763778:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077127.763789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077127.763795:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077127/real 1769077127] req@ffff9b0352a1a300 x1855007617350144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077127.763802:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617350144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077127.763807:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077127.763809:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077127.763811:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077127.763814:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077127.763823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077127.763828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077127/real 1769077127] req@ffff9b0352a19f80 x1855007617350272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077127.763835:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617350272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077127.763840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077127.763841:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077127.763843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077128.275366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077128.659355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077128.659360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077128.659429:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077129.299388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077129.683297:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077129.683377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077129.683379:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077130.323342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077130.707314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077130.707319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077130.707352:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077131.347374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077131.731318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077131.731319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077131.731382:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077132.371348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077132.755325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077132.755402:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077132.755520:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077132.883355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077132.883365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077132.883370:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077132.883378:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4122 00000100:00080000:2.0:1769077132.883388:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077132.883393:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077132.883425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617350400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077132.883450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077132.883453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077132.883454:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077132.883457:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4122 00000100:00080000:2.0:1769077132.883460:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077132.883463:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077132.883471:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077132.883483:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617350528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077132.883507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077132.883510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077132.883512:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077132.883520:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617350400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077132.883521:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4122 00000100:00080000:2.0:1769077132.883530:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077132.883536:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077132.883540:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077132.883553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617350656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077132.883560:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077132.883563:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077132.883564:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077132.883567:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4122 00000100:00080000:2.0:1769077132.883569:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077132.883571:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077132.883575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077132.883581:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617350784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077132.883587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077132.883590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077132.883592:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077132.883596:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4122 00000100:00080000:2.0:1769077132.883598:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077132.883600:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077132.883604:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769077132.883609:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077132.883614:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617350912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077132.883660:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077132/real 1769077132] req@ffff9b03482c4380 x1855007617350400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077132.883677:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617350400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077132.883691:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077132.883695:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077132.883699:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617350400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077132.883723:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077132.883740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077132.883749:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077132/real 1769077132] req@ffff9b03482c5180 x1855007617350528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077132.883757:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617350528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077132.883763:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077132.883765:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077132.883768:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077132.883772:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077132.883782:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077132.883787:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077132/real 1769077132] req@ffff9b03482c4a80 x1855007617350656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077132.883795:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617350656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077132.883801:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077132.883802:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077132.883804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077132.883809:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077132.883820:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077132.883826:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077132/real 1769077132] req@ffff9b03482c5500 x1855007617350784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077132.883833:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617350784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077132.883847:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077132.883849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077132.883851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077132.883856:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077132.883869:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077132.883876:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077132/real 1769077132] req@ffff9b03482c6300 x1855007617350912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077132.883883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617350912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077132.883889:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077132.883891:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077132.883892:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617350912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077133.395336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077133.779317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077133.779361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077133.779436:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077134.419342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077134.803344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077134.803345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077134.803418:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077135.443375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077135.827306:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077135.827306:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077135.827405:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077136.467371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077136.851331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077136.851386:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077136.851386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077137.491345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077137.875353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077137.875353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077137.875434:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077138.003363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077138.003371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077138.003376:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077138.003385:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4127 00000100:00080000:0.0:1769077138.003396:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077138.003417:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077138.003450:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617351040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077138.003474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077138.003478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077138.003480:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077138.003485:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4127 00000100:00080000:0.0:1769077138.003489:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077138.003495:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077138.003504:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077138.003521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617351168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077138.003523:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617351040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077138.003538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077138.003541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077138.003542:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077138.003546:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4127 00000100:00080000:0.0:1769077138.003549:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077138.003552:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077138.003556:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077138.003566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617351296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077138.003578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077138.003580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077138.003582:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077138.003584:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4127 00000100:00080000:0.0:1769077138.003586:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077138.003589:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077138.003594:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769077138.003595:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077138.003606:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617351424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077138.003613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077138.003615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077138.003617:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077138.003619:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4127 00000100:00080000:0.0:1769077138.003622:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077138.003624:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077138.003628:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077138.003632:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff9b0365a3aa00 x1855007617351040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077138.003638:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617351552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077138.003661:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617351040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077138.003671:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077138.003675:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077138.003678:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617351040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077138.003698:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077138.003724:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077138.003744:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff9b0365a3b480 x1855007617351168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077138.003761:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617351168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077138.003769:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077138.003772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077138.003774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077138.003781:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077138.003802:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077138.003813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff9b0365a38e00 x1855007617351296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077138.003821:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617351296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077138.003830:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077138.003833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077138.003837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077138.003843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077138.003860:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077138.003872:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff9b0365a39c00 x1855007617351424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077138.003883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617351424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077138.003888:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077138.003892:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077138.003896:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077138.003901:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077138.003912:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077138.003920:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077138/real 1769077138] req@ffff9b0365a38a80 x1855007617351552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077138.003989:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617351552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077138.003997:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077138.004000:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077138.004003:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077138.515339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077138.899335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077138.899336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077138.899419:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077139.539319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077139.923339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077139.923339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077139.923435:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077140.563402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077140.947345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077140.947345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077140.947432:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077141.587319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077141.971356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077141.971356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077141.971436:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077142.611358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077142.995337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077142.995337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077142.995401:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077143.123370:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077143.123379:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077143.123383:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077143.123391:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4132 00000100:00080000:3.0:1769077143.123399:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077143.123405:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077143.123440:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617351680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077143.123464:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077143.123467:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077143.123468:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077143.123471:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4132 00000100:00080000:3.0:1769077143.123474:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077143.123477:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077143.123484:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077143.123494:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617351808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077143.123500:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617351680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077143.123505:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077143.123508:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077143.123509:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077143.123512:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4132 00000100:00080000:3.0:1769077143.123514:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077143.123516:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077143.123519:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077143.123528:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617351936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077143.123534:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077143.123536:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077143.123538:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077143.123541:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4132 00000100:00080000:3.0:1769077143.123543:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077143.123546:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077143.123549:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077143.123556:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617352064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077143.123563:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077143.123569:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077143.123572:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077143.123574:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077143.123578:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4132 00000100:00080000:3.0:1769077143.123580:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077143.123583:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077143.123586:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077143.123595:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617352192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077143.123604:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077143/real 1769077143] req@ffff9b0352a19f80 x1855007617351680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077143.123624:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617351680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077143.123632:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077143.123635:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077143.123638:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617351680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077143.123656:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077143.123674:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077143.123686:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077143/real 1769077143] req@ffff9b0352a1a300 x1855007617351808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077143.123699:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617351808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077143.123708:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077143.123711:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077143.123715:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077143.123722:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077143.123743:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077143.123751:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077143/real 1769077143] req@ffff9b0352a18e00 x1855007617351936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077143.123759:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617351936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077143.123769:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077143.123771:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077143.123773:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617351936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077143.123777:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077143.123789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077143.123795:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077143/real 1769077143] req@ffff9b0352a1aa00 x1855007617352064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077143.123802:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617352064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077143.123807:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077143.123809:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077143.123810:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077143.123814:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077143.123823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077143.123828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077143/real 1769077143] req@ffff9b0352a18700 x1855007617352192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077143.123835:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617352192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077143.123840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077143.123842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077143.123844:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077143.635360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077144.019364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077144.019400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077144.019432:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077144.659370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077145.043341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077145.043346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077145.043417:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077145.683355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077146.067418:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077146.067482:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077146.067502:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077146.707348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077147.091321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077147.091326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077147.091386:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077147.731336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077148.115351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077148.115352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077148.115446:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077148.243362:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077148.243372:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077148.243377:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077148.243387:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4137 00000100:00080000:3.0:1769077148.243397:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077148.243446:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077148.243479:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617352320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077148.243505:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077148.243508:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077148.243509:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077148.243513:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4137 00000100:00080000:3.0:1769077148.243515:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077148.243518:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077148.243525:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077148.243538:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617352448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077148.243550:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077148.243554:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077148.243559:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077148.243560:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617352320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077148.243566:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4137 00000100:00080000:3.0:1769077148.243572:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077148.243576:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077148.243583:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077148.243598:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617352576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077148.243617:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077148.243620:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077148.243621:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077148.243624:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4137 00000100:00080000:3.0:1769077148.243627:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077148.243630:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077148.243634:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769077148.243638:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077148.243647:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617352704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077148.243654:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077148.243657:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077148.243659:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077148.243661:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4137 00000100:00080000:3.0:1769077148.243663:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077148.243666:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077148.243677:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077148.243684:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617352832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077148.243695:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077148/real 1769077148] req@ffff9b0352a1a680 x1855007617352320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077148.243716:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617352320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077148.243727:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077148.243732:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077148.243737:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617352320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077148.243759:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077148.243781:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077148.243792:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077148/real 1769077148] req@ffff9b0352a18a80 x1855007617352448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077148.243806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617352448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077148.243815:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077148.243842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077148.243850:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077148.243856:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077148.243875:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077148.243886:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077148/real 1769077148] req@ffff9b0352a1b480 x1855007617352576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077148.243900:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617352576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077148.243909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077148.243913:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077148.243917:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077148.243923:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077148.243939:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077148.243948:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077148/real 1769077148] req@ffff9b0352a1ad80 x1855007617352704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077148.243962:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617352704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077148.243972:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077148.243976:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077148.243980:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077148.243986:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077148.244000:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077148.244009:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077148/real 1769077148] req@ffff9b0352a1b100 x1855007617352832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077148.244023:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617352832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077148.244032:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077148.244036:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077148.244040:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617352832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077148.755374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077149.139347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077149.139351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077149.139472:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077149.779421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077150.163368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077150.163368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077150.163452:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077150.803337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077151.187332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077151.187377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077151.187386:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077151.827345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077152.211337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077152.211337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077152.211399:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077152.851343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077153.235295:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077153.235298:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077153.235378:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077153.363415:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077153.363424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077153.363428:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077153.363435:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4142 00000100:00080000:0.0:1769077153.363446:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077153.363484:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077153.363525:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617352960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077153.363551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077153.363554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077153.363556:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077153.363559:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4142 00000100:00080000:0.0:1769077153.363562:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077153.363567:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077153.363574:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077153.363587:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617353088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077153.363588:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617352960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077153.363603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077153.363606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077153.363608:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077153.363612:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4142 00000100:00080000:0.0:1769077153.363614:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077153.363617:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077153.363620:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077153.363629:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617353216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077153.363635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077153.363638:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077153.363639:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077153.363642:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4142 00000100:00080000:0.0:1769077153.363644:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077153.363646:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077153.363649:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077153.363656:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077153.363658:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617353344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077153.363666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077153.363669:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077153.363671:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077153.363674:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4142 00000100:00080000:0.0:1769077153.363678:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077153.363680:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077153.363683:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077153.363691:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617353472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077153.363704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077153/real 1769077153] req@ffff9b0365a38a80 x1855007617352960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077153.363721:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617352960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077153.363730:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077153.363733:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077153.363736:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617352960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077153.363771:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077153.363816:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077153.363827:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077153/real 1769077153] req@ffff9b0365a39c00 x1855007617353088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077153.363838:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617353088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077153.363846:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077153.363849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077153.363851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077153.363858:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077153.363872:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077153.363879:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077153/real 1769077153] req@ffff9b0365a38e00 x1855007617353216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077153.363889:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617353216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077153.363895:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077153.363897:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077153.363899:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077153.363904:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077153.363917:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077153.363933:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077153/real 1769077153] req@ffff9b0365a3b480 x1855007617353344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077153.363944:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617353344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077153.363950:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077153.363953:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077153.363956:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077153.363960:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077153.363973:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077153.363980:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077153/real 1769077153] req@ffff9b0365a3aa00 x1855007617353472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077153.363990:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617353472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077153.363996:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077153.363998:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077153.364001:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077153.875396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077154.259321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077154.259335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077154.259367:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077154.899365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077155.283396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077155.283468:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077155.283520:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077155.923389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077156.307358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077156.307393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077156.307432:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077156.947372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077157.331371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077157.331371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077157.331456:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077157.971344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077158.355347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077158.355390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077158.355419:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077158.483416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077158.483426:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077158.483430:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077158.483438:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4147 00000100:00080000:2.0:1769077158.483448:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077158.483459:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077158.483500:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617353600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077158.483532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077158.483536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077158.483537:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077158.483542:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4147 00000100:00080000:2.0:1769077158.483546:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077158.483551:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077158.483560:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077158.483574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617353728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077158.483587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077158.483590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077158.483592:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077158.483612:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617353600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077158.483615:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4147 00000100:00080000:2.0:1769077158.483622:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077158.483627:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077158.483631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077158.483643:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617353856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077158.483650:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077158.483652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077158.483654:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077158.483657:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4147 00000100:00080000:2.0:1769077158.483668:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077158.483671:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077158.483675:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077158.483680:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077158.483686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617353984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077158.483695:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077158.483697:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077158.483699:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077158.483701:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4147 00000100:00080000:2.0:1769077158.483704:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077158.483706:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077158.483710:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077158.483719:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077158/real 1769077158] req@ffff9b03482c6d80 x1855007617353600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077158.483722:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617354112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077158.483754:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617353600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077158.483781:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077158.483785:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077158.483789:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617353600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077158.483811:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077158.483837:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077158.483852:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077158/real 1769077158] req@ffff9b03482c6680 x1855007617353728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077158.483868:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617353728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077158.483880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077158.483884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077158.483888:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077158.483896:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077158.483916:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077158.483925:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077158/real 1769077158] req@ffff9b03482c7100 x1855007617353856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077158.483934:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617353856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077158.483941:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077158.483944:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077158.483946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077158.483950:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077158.483962:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077158.483969:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077158/real 1769077158] req@ffff9b03482c4000 x1855007617353984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077158.485353:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617353984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077158.485358:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077158.485360:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077158.485362:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617353984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077158.485366:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617354112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077158.485389:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077158.485400:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077158/real 1769077158] req@ffff9b03482c6a00 x1855007617354112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077158.485412:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617354112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077158.485421:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077158.485424:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077158.485427:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617354112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077158.995387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077159.379548:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077159.379811:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077159.380316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077160.019351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077160.403369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077160.403606:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077160.403760:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077161.043327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077161.427443:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077161.427566:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077161.427751:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077162.067359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077162.451396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077162.451491:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077162.451610:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077163.091303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077163.475378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077163.475469:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077163.475523:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769077163.603321:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077163.603332:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077163.603337:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077163.603348:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4152 00000100:00080000:2.0:1769077163.603358:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077163.603365:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077163.603399:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617354240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077163.603426:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077163.603429:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077163.603431:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077163.603435:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4152 00000100:00080000:2.0:1769077163.603438:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077163.603442:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077163.603451:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077163.603462:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617354368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077163.603474:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077163.603478:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077163.603480:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077163.603484:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4152 00000100:00080000:2.0:1769077163.603490:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077163.603491:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617354240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077163.603496:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077163.603505:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077163.603518:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617354496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077163.603534:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077163.603537:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077163.603539:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077163.603543:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4152 00000100:00080000:2.0:1769077163.603555:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077163.603559:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077163.603568:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769077163.603574:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077163.603588:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617354624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077163.603597:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077163.603601:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077163.603603:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077163.603608:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4152 00000100:00080000:2.0:1769077163.603612:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077163.603617:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077163/real 1769077163] req@ffff9b03482c4700 x1855007617354240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077163.603620:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077163.603629:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077163.603647:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617354752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077163.603652:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617354240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077163.603674:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077163.603679:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077163.603684:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617354240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077163.603695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617354368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077163.603723:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077163.603737:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077163/real 1769077163] req@ffff9b03482c5880 x1855007617354368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077163.603752:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617354368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077163.603762:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077163.603766:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077163.603770:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617354368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077163.603796:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617354496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077163.603869:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077163.603883:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077163/real 1769077163] req@ffff9b03482c7b80 x1855007617354496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077163.603898:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617354496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077163.603908:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077163.603912:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077163.603915:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617354496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077163.603922:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617354624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077163.603943:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077163.603955:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077163/real 1769077163] req@ffff9b03482c5f80 x1855007617354624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077163.603969:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617354624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077163.603995:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077163.603999:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077163.604002:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617354624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077163.604009:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617354752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077163.604029:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077163.604041:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077163/real 1769077163] req@ffff9b03482c5c00 x1855007617354752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077163.604055:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617354752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077163.604070:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077163.604074:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077163.604077:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617354752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077164.115415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077164.499583:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077164.499583:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077164.500054:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077165.139380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077165.523329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077165.523439:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077165.523474:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077166.163338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077166.547366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077166.547391:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077166.547398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077167.187338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077167.571352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077167.571352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077167.571465:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077168.211346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077168.595319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077168.595345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077168.595411:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077168.723337:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077168.723349:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077168.723354:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077168.723365:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4157 00000100:00080000:2.0:1769077168.723373:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077168.723379:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077168.723412:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617354880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077168.723440:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077168.723443:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077168.723445:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077168.723448:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4157 00000100:00080000:2.0:1769077168.723450:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077168.723453:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077168.723460:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077168.723471:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617355008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077168.723479:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077168.723482:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077168.723483:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077168.723486:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4157 00000100:00080000:2.0:1769077168.723489:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077168.723491:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077168.723496:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077168.723503:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617355136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077168.723513:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617354880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077168.723514:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077168.723519:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077168.723522:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077168.723528:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4157 00000100:00080000:2.0:1769077168.723531:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077168.723533:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077168.723536:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077168.723545:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617355264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077168.723552:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077168.723555:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077168.723556:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077168.723559:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4157 00000100:00080000:2.0:1769077168.723561:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077168.723563:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077168.723566:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077168.723573:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617355392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077168.723593:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077168.723629:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077168/real 1769077168] req@ffff9b03482c4e00 x1855007617354880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077168.723644:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617354880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077168.723652:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077168.723656:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077168.723659:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617354880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077168.723665:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077168.723681:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077168.723689:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077168/real 1769077168] req@ffff9b03482c5c00 x1855007617355008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077168.723717:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617355008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077168.723724:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077168.723726:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077168.723729:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077168.723750:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077168.723763:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077168.723770:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077168/real 1769077168] req@ffff9b03482c5f80 x1855007617355136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077168.723780:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617355136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077168.723786:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077168.723788:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077168.723790:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077168.723794:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077168.723805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077168.723811:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077168/real 1769077168] req@ffff9b03482c7b80 x1855007617355264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077168.723820:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617355264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077168.723831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077168.723833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077168.723835:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077168.723839:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077168.723849:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077168.723855:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077168/real 1769077168] req@ffff9b03482c5880 x1855007617355392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077168.723863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617355392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077168.723868:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077168.723871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077168.723873:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077169.235341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077169.619382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077169.619434:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077169.619486:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077170.259351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077170.643310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077170.643317:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077170.643337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077171.283348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077171.667367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077171.667367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077171.667428:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077172.307342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077172.691349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077172.691382:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077172.691392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077173.331338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077173.715313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077173.715380:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077173.715380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769077173.843464:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077173.843478:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077173.843495:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077173.843512:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4163 00000100:00080000:3.0:1769077173.843526:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077173.843535:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077173.843589:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617355520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077173.843655:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077173.843660:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077173.843663:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077173.843670:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4163 00000100:00080000:3.0:1769077173.843676:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077173.843681:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077173.843695:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077173.843724:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617355648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077173.843733:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077173.843736:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077173.843738:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077173.843742:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4163 00000100:00080000:3.0:1769077173.843746:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077173.843750:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077173.843779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617355520:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769077173.843862:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077173.843876:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617355776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077173.843901:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077173.843906:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1769077173.843909:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077173.843909:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077173.843917:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4163 00000100:00080000:3.0:1769077173.843922:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077173.843927:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077173.843934:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077173.843967:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077173/real 1769077173] req@ffff9b0352a18000 x1855007617355520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077173.843998:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617355520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077173.844009:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077173.844018:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077173.844023:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617355520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077173.844039:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077173.844070:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077173.844083:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077173/real 1769077173] req@ffff9b0352a19c00 x1855007617355648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077173.844105:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617355648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077173.844125:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617355904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077173.844126:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077173.844146:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077173.844152:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355648:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769077173.844156:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077173.844161:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769077173.844183:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077173.844203:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077173.844223:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077173/real 1769077173] req@ffff9b0352a1b800 x1855007617355776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077173.844264:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077173.844265:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617355776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077173.844276:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4163 00000100:00080000:3.0:1769077173.844287:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077173.844292:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077173.844295:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:1.0:1769077173.844301:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000008:00100000:3.0:1769077173.844304:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077173.844311:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077173.844323:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077173.844329:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617356032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077173.844354:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077173.844368:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077173/real 1769077173] req@ffff9b0352a19500 x1855007617355904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077173.844377:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617355904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077173.844383:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077173.844386:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077173.844388:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617355904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077173.844408:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077173.844418:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077173.844424:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077173/real 1769077173] req@ffff9b0352a18380 x1855007617356032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077173.844431:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617356032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077173.844436:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077173.844439:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077173.844442:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077174.355319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077174.739364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077174.739412:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077174.739460:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077175.379418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077175.763356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077175.763384:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077175.763483:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077176.403327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077176.787307:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077176.787307:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077176.787358:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077177.427322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077177.811301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077177.811324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077177.811379:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077178.451355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077178.835339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077178.835370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077178.835421:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077178.963352:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077178.963365:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077178.963370:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077178.963378:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4168 00000100:00080000:3.0:1769077178.963386:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077178.963392:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077178.963438:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617356160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077178.963462:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077178.963465:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077178.963477:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077178.963481:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4168 00000100:00080000:3.0:1769077178.963484:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077178.963487:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077178.963494:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077178.963506:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617356160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077178.963506:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617356288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077178.963522:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077178.963524:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077178.963526:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077178.963529:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4168 00000100:00080000:3.0:1769077178.963532:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077178.963546:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077178.963549:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077178.963558:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617356416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077178.963565:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077178.963567:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077178.963568:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077178.963571:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4168 00000100:00080000:3.0:1769077178.963573:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077178.963576:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077178.963580:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077178.963587:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617356544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077178.963594:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077178.963596:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077178.963598:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077178.963600:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4168 00000100:00080000:3.0:1769077178.963602:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077178.963605:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077178.963609:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077178.963617:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617356672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077178.963622:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077178.963649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077178.963662:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077178.963681:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077179.475281:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077179.859355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077179.859388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077179.859419:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077180.499319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077180.883350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077180.883351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077180.883418:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077181.523301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077181.907343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077181.907354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077181.907396:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769077182.035648:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769077182.035722:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769077182.035740:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077182.035793:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077182.035805:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077182.035818:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077182.035830:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769077182.035849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077178/real 1769077182] req@ffff9b0352a19500 x1855007617356288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077182.035878:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617356288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077182.035892:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077182.035899:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077182.035905:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077182.035914:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077178/real 1769077182] req@ffff9b0352a1b800 x1855007617356416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077182.035932:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617356416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077182.035942:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077182.035946:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077182.035950:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077182.035958:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077178/real 1769077182] req@ffff9b0352a19c00 x1855007617356544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077182.035972:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617356544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077182.035980:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077182.035984:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077182.035988:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077182.035996:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077178/real 1769077182] req@ffff9b0352a18380 x1855007617356160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077182.036019:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617356160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077182.036028:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077182.036032:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077182.036037:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617356160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077182.036067:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077178/real 1769077182] req@ffff9b0352a18000 x1855007617356672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077182.036083:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617356672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077182.036093:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077182.036096:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077182.036100:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077182.547293:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077182.931382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077182.931382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077182.931470:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077183.571445:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077183.955334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077183.955362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077183.955364:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077184.083446:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077184.083463:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077184.083469:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077184.083481:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4173 00000100:00080000:2.0:1769077184.083493:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077184.083500:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077184.083537:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617356800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077184.083564:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077184.083568:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077184.083570:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077184.083575:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4173 00000100:00080000:2.0:1769077184.083580:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077184.083585:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077184.083598:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077184.083623:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617356928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077184.083639:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617356800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077184.083662:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077184.083669:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077184.083672:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077184.083681:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4173 00000100:00080000:2.0:1769077184.083686:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077184.083691:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077184.083697:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077184.083715:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617357056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077184.083725:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077184.083729:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077184.083731:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769077184.083739:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077184.083740:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4173 00000100:00080000:2.0:1769077184.083750:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077184.083757:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077184.083765:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077184.083782:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617357184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077184.083794:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077184.083801:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077184/real 1769077184] req@ffff9b03482c4700 x1855007617356800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077184.083803:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077184.083808:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077184.083823:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4173 00000100:00080000:2.0:1769077184.083836:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077184.083843:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077184.083850:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617356800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769077184.083853:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769077184.083879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077184.083892:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077184.083894:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617357312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077184.083904:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617356800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077184.083954:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077184.083985:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077184.084002:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077184/real 1769077184] req@ffff9b03482c6a00 x1855007617356928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077184.084032:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617356928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077184.084042:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077184.084046:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077184.084050:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617356928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077184.084057:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077184.084104:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077184.084117:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077184/real 1769077184] req@ffff9b03482c4000 x1855007617357056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077184.084129:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617357056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077184.084138:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077184.084142:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077184.084145:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077184.084151:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077184.084173:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077184.084184:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077184/real 1769077184] req@ffff9b03482c7100 x1855007617357184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077184.084196:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617357184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077184.084210:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077184.084214:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077184.084217:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077184.084225:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077184.084273:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077184.084287:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077184/real 1769077184] req@ffff9b03482c6680 x1855007617357312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077184.084299:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617357312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077184.084308:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077184.084311:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077184.084314:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077184.595318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077184.979335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077184.979335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077184.979557:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077185.619301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077186.003321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077186.003321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077186.003381:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077186.643305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077187.027347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077187.027373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077187.027440:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077187.667319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077188.051326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077188.051327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077188.051396:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077188.691306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077189.075359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077189.075359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077189.075414:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077189.203332:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077189.203346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077189.203352:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077189.203365:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4178 00000100:00080000:3.0:1769077189.203376:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077189.203385:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077189.203450:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617357440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077189.203483:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077189.203488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077189.203490:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077189.203495:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4178 00000100:00080000:3.0:1769077189.203501:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077189.203507:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077189.203519:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077189.203527:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617357440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077189.203536:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617357568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077189.203588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077189.203593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077189.203595:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077189.203600:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4178 00000100:00080000:3.0:1769077189.203605:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077189.203609:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077189.203615:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077189.203631:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617357696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077189.203641:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769077189.203646:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077189.203648:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077189.203652:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077189.203658:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4178 00000100:00080000:3.0:1769077189.203664:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077189.203668:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077189.203674:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077189.203686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617357824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077189.203702:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077189/real 1769077189] req@ffff9b0352a1b100 x1855007617357440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077189.203707:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077189.203716:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077189.203720:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077189.203727:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4178 00000100:00080000:3.0:1769077189.203731:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077189.203738:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617357440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077189.203738:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077189.203747:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769077189.203753:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077189.203758:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077189.203763:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617357952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077189.203764:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617357440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077189.203796:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077189.203817:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077189.203826:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077189/real 1769077189] req@ffff9b0352a1ad80 x1855007617357568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077189.203835:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617357568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077189.203842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077189.203844:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077189.203846:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077189.203850:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077189.203861:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077189.203867:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077189/real 1769077189] req@ffff9b0352a1b480 x1855007617357696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077189.203879:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617357696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077189.203885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077189.203886:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077189.203888:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077189.203892:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077189.203902:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077189.203908:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077189/real 1769077189] req@ffff9b0352a18a80 x1855007617357824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077189.203915:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617357824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077189.203920:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077189.203922:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077189.203923:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077189.203927:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077189.203936:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077189.203941:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077189/real 1769077189] req@ffff9b0352a1a680 x1855007617357952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077189.203947:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617357952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077189.203957:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077189.203959:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077189.203961:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617357952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077189.715326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077190.099333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077190.099333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077190.099385:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077190.739323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077191.123345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077191.123350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077191.123399:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077191.763352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077192.147378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077192.147379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077192.147452:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077192.787374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077193.171374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077193.171381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077193.171459:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077193.811327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769077194.195386:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077194.195390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077194.195391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769077194.323327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077194.323337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077194.323341:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077194.323350:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4183 00000100:00080000:3.0:1769077194.323359:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077194.323365:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077194.323397:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617358080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077194.323419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077194.323423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077194.323424:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077194.323427:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4183 00000100:00080000:3.0:1769077194.323429:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077194.323432:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077194.323440:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077194.323451:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617358080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077194.323451:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617358208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077194.323475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077194.323480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077194.323482:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077194.323485:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4183 00000100:00080000:3.0:1769077194.323487:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077194.323490:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077194.323494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077194.323502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617358336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077194.323509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077194.323512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077194.323513:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077194.323516:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4183 00000100:00080000:3.0:1769077194.323518:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077194.323520:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077194.323525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077194.323532:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617358464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077194.323537:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077194.323544:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077194.323547:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077194.323548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077194.323552:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4183 00000100:00080000:3.0:1769077194.323555:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077194.323557:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077194.323560:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077194.323567:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617358592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077194.323588:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077194/real 1769077194] req@ffff9b0352a18700 x1855007617358080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077194.323604:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617358080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077194.323612:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077194.323615:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077194.323618:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617358080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077194.323636:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077194.323653:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077194.323662:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077194/real 1769077194] req@ffff9b0352a1aa00 x1855007617358208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077194.323670:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617358208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077194.323675:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077194.323677:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077194.323679:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077194.323683:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077194.323707:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077194.323713:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077194/real 1769077194] req@ffff9b0352a18e00 x1855007617358336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077194.323721:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617358336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077194.323726:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077194.323728:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077194.323730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077194.323737:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077194.323747:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077194.323753:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077194/real 1769077194] req@ffff9b0352a1a300 x1855007617358464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077194.323760:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617358464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077194.323766:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077194.323769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077194.323771:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077194.323774:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077194.323783:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077194.323788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077194/real 1769077194] req@ffff9b0352a19f80 x1855007617358592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077194.323795:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617358592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077194.323800:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077194.323802:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077194.323804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077194.835339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077195.219317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077195.219318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077195.219402:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077195.859352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077196.243348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077196.243372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077196.243412:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077196.883398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077197.267354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077197.267354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077197.267461:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077197.907384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077198.291362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077198.291374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077198.291465:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077198.931430:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077199.315341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077199.315411:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077199.315441:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077199.443401:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077199.443427:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077199.443432:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077199.443442:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4188 00000100:00080000:3.0:1769077199.443454:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077199.443462:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077199.443504:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617358720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077199.443533:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077199.443536:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077199.443538:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077199.443541:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4188 00000100:00080000:3.0:1769077199.443544:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077199.443548:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077199.443557:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077199.443569:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617358848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077199.443582:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617358720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077199.443582:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077199.443599:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077199.443602:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077199.443606:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4188 00000100:00080000:3.0:1769077199.443608:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077199.443610:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077199.443613:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077199.443622:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617358976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077199.443630:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077199.443633:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077199.443634:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077199.443638:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4188 00000100:00080000:3.0:1769077199.443641:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077199.443643:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077199.443647:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077199.443656:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617359104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077199.443662:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077199.443664:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077199.443665:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077199.443675:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4188 00000100:00080000:3.0:1769077199.443678:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077199.443681:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769077199.443685:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769077199.443686:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077199.443697:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617359232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077199.443747:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077199/real 1769077199] req@ffff9b0352a19f80 x1855007617358720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077199.443770:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617358720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077199.443780:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077199.443784:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077199.443787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617358720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077199.443815:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077199.443833:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077199.443842:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077199/real 1769077199] req@ffff9b0352a1a300 x1855007617358848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077199.443851:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617358848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077199.443858:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077199.443860:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077199.443862:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077199.443867:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077199.443878:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077199.443884:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077199/real 1769077199] req@ffff9b0352a18e00 x1855007617358976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077199.443892:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617358976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077199.443902:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077199.443905:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077199.443907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617358976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077199.443911:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077199.443921:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077199.443927:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077199/real 1769077199] req@ffff9b0352a1aa00 x1855007617359104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077199.443934:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617359104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077199.443940:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077199.443942:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077199.443944:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077199.443947:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077199.443964:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077199.443971:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077199/real 1769077199] req@ffff9b0352a18700 x1855007617359232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077199.443979:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617359232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077199.443984:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077199.443986:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077199.443989:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077199.955411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077200.339380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077200.339386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077200.339467:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077200.979416:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077201.363411:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077201.363436:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077201.363488:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077202.003313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077202.387370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077202.387381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077202.387433:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077203.027367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077203.411361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077203.411422:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077203.411436:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077204.051335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077204.435373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077204.435408:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077204.435456:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077204.563346:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077204.563356:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077204.563360:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077204.563368:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4193 00000100:00080000:2.0:1769077204.563394:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077204.563401:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077204.563436:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617359360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077204.563472:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077204.563476:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077204.563478:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077204.563482:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4193 00000100:00080000:2.0:1769077204.563485:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077204.563488:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077204.563497:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077204.563529:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617359488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077204.563536:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617359360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077204.563539:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077204.563544:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077204.563545:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077204.563548:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4193 00000100:00080000:2.0:1769077204.563550:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077204.563553:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077204.563557:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077204.563565:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617359616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077204.563571:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077204.563573:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077204.563574:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077204.563576:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4193 00000100:00080000:2.0:1769077204.563579:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077204.563581:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077204.563585:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077204.563596:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617359744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077204.563597:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077204.563605:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077204.563608:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077204.563609:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077204.563612:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4193 00000100:00080000:2.0:1769077204.563615:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077204.563618:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077204.563621:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077204.563633:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617359872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077204.563645:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077204/real 1769077204] req@ffff9b03482c6d80 x1855007617359360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077204.563666:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617359360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077204.563673:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077204.563677:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077204.563680:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617359360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077204.563698:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077204.563717:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077204.563726:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077204/real 1769077204] req@ffff9b03482c6300 x1855007617359488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077204.563734:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617359488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077204.563741:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077204.563743:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077204.563745:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077204.563749:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077204.563759:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077204.563765:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077204/real 1769077204] req@ffff9b03482c5500 x1855007617359616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077204.563772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617359616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077204.563786:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077204.563788:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077204.563790:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077204.563794:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077204.563807:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077204.563813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077204/real 1769077204] req@ffff9b03482c4a80 x1855007617359744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077204.563821:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617359744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077204.563826:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077204.563829:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077204.563831:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077204.563835:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077204.563844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077204.563849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077204/real 1769077204] req@ffff9b03482c5180 x1855007617359872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077204.563856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617359872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077204.563862:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077204.563864:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077204.563867:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617359872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077205.075375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077205.459358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077205.459413:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077205.459435:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077206.099365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077206.483367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077206.483367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077206.483433:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077207.123344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077207.507323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077207.507323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077207.507408:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077208.147356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077208.531352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077208.531426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077208.531455:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077209.171337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077209.555315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077209.555340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077209.555439:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077209.683370:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077209.683383:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077209.683390:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077209.683402:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4198 00000100:00080000:2.0:1769077209.683412:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077209.683419:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077209.683457:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617360000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077209.683502:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077209.683505:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077209.683508:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077209.683513:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4198 00000100:00080000:2.0:1769077209.683517:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077209.683522:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077209.683533:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077209.683552:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617360000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077209.683554:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617360128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077209.683572:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077209.683587:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077209.683588:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077209.683592:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4198 00000100:00080000:2.0:1769077209.683596:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077209.683600:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077209.683605:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077209.683617:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617360256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077209.683625:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769077209.683629:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077209.683630:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077209.683634:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077209.683639:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4198 00000100:00080000:2.0:1769077209.683642:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077209.683646:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077209.683651:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077209.683662:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617360384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077209.683669:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077209.683672:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077209.683674:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077209.683680:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077209/real 1769077209] req@ffff9b03482c4380 x1855007617360000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077209.683680:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4198 00000100:00080000:2.0:1769077209.683685:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077209.683690:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077209.683696:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077209.683709:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617360512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077209.683719:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617360000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077209.683735:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077209.683741:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077209.683748:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617360000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077209.683775:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077209.683811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077209.683829:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077209/real 1769077209] req@ffff9b03482c5180 x1855007617360128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077209.683841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617360128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077209.683849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077209.683852:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077209.683855:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077209.683860:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077209.683881:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077209.683891:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077209/real 1769077209] req@ffff9b03482c4a80 x1855007617360256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077209.683905:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617360256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077209.683913:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077209.683917:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077209.683920:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077209.683927:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077209.683942:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077209.683949:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077209/real 1769077209] req@ffff9b03482c5500 x1855007617360384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077209.683964:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617360384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077209.683975:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077209.683980:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077209.683988:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077209.683998:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077209.684029:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077209.684048:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077209/real 1769077209] req@ffff9b03482c6300 x1855007617360512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077209.684061:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617360512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077209.684068:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077209.684072:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077209.684076:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077210.195386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077210.579370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077210.579370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077210.579445:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077211.219379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077211.603393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077211.603429:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077211.603506:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077212.243357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077212.627386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077212.627386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077212.627439:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077213.267328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077213.651386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077213.651388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077213.651443:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077214.291349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077214.675384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077214.675393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077214.675462:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077214.803362:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077214.803373:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077214.803377:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077214.803386:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4203 00000100:00080000:3.0:1769077214.803394:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077214.803400:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077214.803434:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617360640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077214.803461:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077214.803471:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077214.803473:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077214.803477:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4203 00000100:00080000:3.0:1769077214.803480:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077214.803485:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077214.803493:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077214.803519:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617360768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077214.803525:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617360640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077214.803531:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077214.803535:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077214.803536:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077214.803539:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4203 00000100:00080000:3.0:1769077214.803543:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077214.803546:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077214.803550:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077214.803561:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617360896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077214.803568:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077214.803571:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077214.803572:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077214.803574:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4203 00000100:00080000:3.0:1769077214.803577:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077214.803579:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077214.803583:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769077214.803591:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077214.803592:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617361024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077214.803601:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077214.803604:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077214.803606:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077214.803609:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4203 00000100:00080000:3.0:1769077214.803612:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077214.803614:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077214.803617:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077214.803624:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617361152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077214.803639:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077214/real 1769077214] req@ffff9b0352a1a680 x1855007617360640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077214.803660:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617360640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077214.803671:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077214.803676:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077214.803680:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617360640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077214.803700:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077214.803720:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077214.803729:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077214/real 1769077214] req@ffff9b0352a18a80 x1855007617360768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077214.803747:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617360768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077214.803753:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077214.803755:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077214.803758:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077214.803763:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077214.803775:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077214.803781:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077214/real 1769077214] req@ffff9b0352a1b480 x1855007617360896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077214.803788:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617360896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077214.803793:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077214.803795:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077214.803797:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617360896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077214.803803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077214.803822:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077214.803833:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077214/real 1769077214] req@ffff9b0352a1ad80 x1855007617361024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077214.803846:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617361024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077214.803854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077214.803857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077214.803861:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077214.803893:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077214.803922:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077214.803938:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077214/real 1769077214] req@ffff9b0352a1b100 x1855007617361152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077214.803968:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617361152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077214.803974:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077214.803976:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077214.803978:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077215.315355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077215.699320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077215.699383:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077215.699399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077216.339340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077216.723329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077216.723330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077216.723391:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077217.363357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077217.747315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077217.747324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077217.747397:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077218.387347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077218.771342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077218.771342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077218.771424:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077219.411308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077219.795386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077219.795402:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077219.795491:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077219.923399:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077219.923409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077219.923412:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077219.923420:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4209 00000100:00080000:0.0:1769077219.923429:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077219.923436:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077219.923484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617361280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077219.923510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077219.923514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077219.923515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077219.923519:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4209 00000100:00080000:0.0:1769077219.923521:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077219.923525:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077219.923561:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077219.923572:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617361408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077219.923579:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617361280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077219.923583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077219.923587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077219.923589:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077219.923592:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4209 00000100:00080000:0.0:1769077219.923594:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077219.923597:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077219.923601:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077219.923609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617361536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077219.923615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077219.923617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077219.923619:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077219.923622:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4209 00000100:00080000:0.0:1769077219.923633:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077219.923636:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077219.923640:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769077219.923643:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077219.923652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617361664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077219.923661:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077219.923663:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077219.923664:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077219.923667:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4209 00000100:00080000:0.0:1769077219.923669:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077219.923671:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077219.923675:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077219.923684:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617361792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077219.923688:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077219/real 1769077219] req@ffff9b0365a39500 x1855007617361280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077219.923712:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617361280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077219.923720:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077219.923724:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077219.923727:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617361280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077219.923746:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077219.923762:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077219.923771:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077219/real 1769077219] req@ffff9b0365a39180 x1855007617361408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077219.923790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617361408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077219.923796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077219.923799:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077219.923801:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077219.923824:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077219.923850:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077219.923858:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077219/real 1769077219] req@ffff9b0365a3ad80 x1855007617361536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077219.923867:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617361536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077219.923873:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077219.923876:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077219.923878:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077219.923882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077219.923893:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077219.923900:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077219/real 1769077219] req@ffff9b0365a3bb80 x1855007617361664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077219.923923:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617361664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077219.923933:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077219.923936:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077219.923940:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077219.923948:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077219.923972:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077219.923986:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077219/real 1769077219] req@ffff9b0365a3b100 x1855007617361792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077219.924013:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617361792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077219.924037:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077219.924042:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077219.924046:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617361792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077220.435407:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077220.819347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077220.819348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077220.819511:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077221.459403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077221.843367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077221.843417:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077221.843491:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077222.483393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077222.867409:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077222.867501:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077222.867531:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077223.507324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077223.891405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077223.891406:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077223.891493:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077224.531347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077224.915331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077224.915331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077224.915396:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077225.043434:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077225.043447:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077225.043452:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077225.043464:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4214 00000100:00080000:3.0:1769077225.043472:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077225.043478:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077225.043512:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617361920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077225.043532:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077225.043535:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077225.043536:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077225.043539:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4214 00000100:00080000:3.0:1769077225.043541:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077225.043543:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077225.043550:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077225.043560:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617362048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077225.043568:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077225.043571:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077225.043573:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077225.043577:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617361920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077225.043577:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4214 00000100:00080000:3.0:1769077225.043580:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077225.043586:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077225.043591:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077225.043599:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617362176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077225.043604:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077225.043606:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077225.043608:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077225.043610:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4214 00000100:00080000:3.0:1769077225.043612:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077225.043615:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077225.043618:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077225.043625:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617362304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077225.043630:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077225.043632:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077225.043634:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077225.043638:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4214 00000100:00080000:3.0:1769077225.043640:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077225.043643:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077225.043646:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769077225.043652:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077225.043656:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617362432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077225.043699:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff9b0352a18000 x1855007617361920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077225.043732:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617361920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077225.043744:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077225.043749:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077225.043753:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617361920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077225.043779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077225.043806:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077225.043819:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff9b0352a18380 x1855007617362048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077225.043834:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617362048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077225.043843:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077225.043847:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077225.043851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077225.043858:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077225.043880:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077225.043892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff9b0352a19c00 x1855007617362176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077225.043905:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617362176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077225.043915:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077225.043928:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077225.043932:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077225.043966:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077225.043991:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077225.044003:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff9b0352a1b800 x1855007617362304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077225.044017:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617362304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077225.044026:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077225.044030:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077225.044033:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077225.044040:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077225.044060:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077225.044073:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077225/real 1769077225] req@ffff9b0352a19500 x1855007617362432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077225.044086:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617362432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077225.044112:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077225.044116:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077225.044119:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077225.555358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077225.939486:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077225.939736:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077225.939769:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077226.579329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077226.963408:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077226.963415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077226.963592:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077227.603373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077227.987392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077227.987392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077227.987565:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077228.627337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077229.011347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077229.011347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077229.011444:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077229.651344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077230.035399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077230.035509:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077230.035632:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:2.0:1769077230.163361:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077230.163370:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077230.163374:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077230.163383:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4219 00000100:00080000:2.0:1769077230.163391:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077230.163397:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077230.163430:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617362560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077230.163453:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077230.163456:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077230.163457:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077230.163461:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4219 00000100:00080000:2.0:1769077230.163475:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077230.163478:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077230.163486:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077230.163498:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617362688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077230.163499:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617362560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077230.163525:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077230.163538:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077230.163540:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077230.163543:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4219 00000100:00080000:2.0:1769077230.163546:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077230.163548:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077230.163552:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077230.163560:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617362816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077230.163566:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077230.163569:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077230.163571:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769077230.163575:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077230.163578:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4219 00000100:00080000:2.0:1769077230.163582:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077230.163586:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077230.163590:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077230.163598:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617362944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077230.163605:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077230.163608:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077230.163609:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077230.163629:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4219 00000100:00100000:1.0:1769077230.163630:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077230/real 1769077230] req@ffff9b03482c6d80 x1855007617362560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077230.163634:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077230.163641:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077230.163648:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077230.163656:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617362560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077230.163659:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617363072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077230.163689:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077230.163695:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077230.163700:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617362560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077230.163723:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077230.163759:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077230.163773:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077230/real 1769077230] req@ffff9b03482c6680 x1855007617362688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077230.163788:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617362688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077230.163794:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077230.163797:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077230.163799:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077230.163804:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077230.163819:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077230.163826:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077230/real 1769077230] req@ffff9b03482c7100 x1855007617362816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077230.163834:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617362816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077230.163839:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077230.163842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077230.163844:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077230.163847:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077230.163885:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077230.163893:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077230/real 1769077230] req@ffff9b03482c4000 x1855007617362944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077230.163912:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617362944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077230.163917:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077230.163919:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077230.163921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617362944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077230.163925:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077230.163935:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077230.163941:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077230/real 1769077230] req@ffff9b03482c6a00 x1855007617363072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077230.163948:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617363072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077230.163953:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077230.163955:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077230.163957:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077230.675337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077231.059352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077231.059369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077231.059410:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077231.699354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077232.083369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077232.083373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077232.083446:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077232.723336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077233.107369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077233.107373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077233.107448:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077233.747319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077234.131355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077234.131374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077234.131449:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077234.771336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077235.155337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077235.155338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077235.155431:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077235.283372:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077235.283383:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077235.283388:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077235.283398:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4224 00000100:00080000:3.0:1769077235.283409:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077235.283416:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077235.283451:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617363200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077235.283480:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077235.283484:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077235.283486:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077235.283490:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4224 00000100:00080000:3.0:1769077235.283494:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077235.283501:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077235.283512:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077235.283531:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617363328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077235.283534:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617363200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077235.283559:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077235.283563:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077235.283566:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077235.283574:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4224 00000100:00080000:3.0:1769077235.283578:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077235.283583:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077235.283589:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077235.283605:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617363456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077235.283616:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077235.283620:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077235.283623:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769077235.283633:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077235.283634:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4224 00000100:00080000:3.0:1769077235.283642:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077235.283649:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077235.283657:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077235.283668:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617363584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077235.283677:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077235.283679:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077235.283686:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077235.283687:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077235/real 1769077235] req@ffff9b0352a19500 x1855007617363200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077235.283704:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4224 00000100:00080000:3.0:1769077235.283718:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077235.283724:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077235.283729:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617363200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077235.283729:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769077235.283750:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077235.283750:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617363712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077235.283781:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077235.283791:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617363200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077235.283823:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077235.283844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077235.283854:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077235/real 1769077235] req@ffff9b0352a1b800 x1855007617363328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077235.283862:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617363328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077235.283869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077235.283872:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077235.283874:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077235.283879:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077235.283897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077235.283909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077235/real 1769077235] req@ffff9b0352a19c00 x1855007617363456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077235.283923:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617363456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077235.283933:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077235.283937:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077235.283941:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077235.283947:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077235.283965:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077235.283972:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077235/real 1769077235] req@ffff9b0352a18380 x1855007617363584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077235.283981:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617363584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077235.283986:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077235.283989:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077235.283991:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077235.283995:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077235.284005:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077235.284011:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077235/real 1769077235] req@ffff9b0352a18000 x1855007617363712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077235.284019:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617363712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077235.284025:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077235.284027:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077235.284030:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077235.795341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077236.179334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077236.179334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077236.179398:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077236.819322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077237.203308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077237.203376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077237.203573:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077237.843333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077238.227329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077238.227329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077238.227397:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077238.867330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077239.251347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077239.251403:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077239.251465:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077239.891354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077240.275360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077240.275364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077240.275382:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077240.403412:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077240.403425:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077240.403430:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077240.403441:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4229 00000100:00080000:2.0:1769077240.403452:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077240.403459:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077240.403507:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617363840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077240.403532:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077240.403536:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077240.403537:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077240.403543:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4229 00000100:00080000:2.0:1769077240.403546:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077240.403549:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077240.403563:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077240.403575:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617363968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077240.403577:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617363840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077240.403606:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077240.403608:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077240.403611:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077240.403617:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4229 00000100:00080000:2.0:1769077240.403619:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077240.403622:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077240.403626:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077240.403635:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617364096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077240.403642:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077240.403645:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077240.403648:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077240.403651:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4229 00000100:00080000:2.0:1769077240.403655:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077240.403664:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769077240.403668:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769077240.403678:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077240.403713:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617364224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077240.403728:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077240/real 1769077240] req@ffff9b03482c4700 x1855007617363840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077240.403730:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077240.403750:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077240.403756:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077240.403762:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4229 00000100:00080000:2.0:1769077240.403768:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077240.403771:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617363840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077240.403777:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077240.403792:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769077240.403796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077240.403802:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077240.403811:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617363840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077240.403818:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617364352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077240.403843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077240.403865:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077240.403874:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077240/real 1769077240] req@ffff9b03482c5880 x1855007617363968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077240.403888:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617363968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077240.403894:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077240.403896:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077240.403899:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617363968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077240.403903:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077240.403914:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077240.403919:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077240/real 1769077240] req@ffff9b03482c7b80 x1855007617364096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077240.403927:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617364096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077240.403932:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077240.403934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077240.403936:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077240.403940:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077240.403950:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077240.403955:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077240/real 1769077240] req@ffff9b03482c5f80 x1855007617364224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077240.403963:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617364224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077240.403972:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077240.403974:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077240.403976:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077240.403980:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077240.403989:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077240.403995:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077240/real 1769077240] req@ffff9b03482c5c00 x1855007617364352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077240.404010:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617364352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077240.404016:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077240.404018:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077240.404020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077240.915309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077241.299330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077241.299339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077241.299444:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077241.939348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077242.323351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077242.323406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077242.323447:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077242.963340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077243.347355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077243.347373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077243.347417:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077243.987364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077244.371310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077244.371389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077244.371407:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077245.011342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077245.395341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077245.395341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077245.395416:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077245.523320:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077245.523330:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077245.523334:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077245.523342:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4234 00000100:00080000:2.0:1769077245.523350:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077245.523358:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077245.523392:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617364480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077245.523430:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077245.523435:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077245.523437:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077245.523442:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4234 00000100:00080000:2.0:1769077245.523447:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077245.523451:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077245.523460:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077245.523463:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617364480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077245.523475:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617364608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077245.523486:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077245.523489:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077245.523491:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077245.523495:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4234 00000100:00080000:2.0:1769077245.523499:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077245.523502:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077245.523507:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077245.523517:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617364736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077245.523524:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077245.523527:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077245.523528:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077245.523531:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4234 00000100:00080000:2.0:1769077245.523534:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077245.523537:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077245.523542:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077245.523551:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617364864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077245.523559:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077245.523561:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077245.523563:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077245.523566:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4234 00000100:00080000:2.0:1769077245.523569:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077245.523571:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364608:192.168.204.160@tcp:38:kworker.0 00000100:00080000:2.0:1769077245.523575:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077245.523580:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077245.523590:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617364992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077245.523677:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077245.523697:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077245.523713:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077246.035289:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077246.419348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077246.419348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077246.419395:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077247.059313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077247.443343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077247.443343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077247.443418:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077248.083293:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077248.467334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077248.467388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077248.467410:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769077248.595570:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769077248.595629:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077248.595649:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077248.595695:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077248.595716:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077248.595724:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077248.595731:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:3.0:1769077248.595777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077245/real 1769077248] req@ffff9b03482c5880 x1855007617364992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077248.595806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617364992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077248.595820:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077248.595825:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077248.595830:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077248.595839:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077245/real 1769077248] req@ffff9b03482c5c00 x1855007617364608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077248.595853:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617364608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077248.595862:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077248.595866:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077248.595871:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077248.595878:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077245/real 1769077248] req@ffff9b03482c5f80 x1855007617364736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077248.595891:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617364736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077248.595900:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077248.595904:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077248.595907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077248.595918:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077245/real 1769077248] req@ffff9b03482c7b80 x1855007617364864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077248.595930:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617364864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077248.595939:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077248.595942:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077248.595946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617364864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077248.595954:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077245/real 1769077248] req@ffff9b03482c4e00 x1855007617364480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077248.595967:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617364480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077248.595975:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077248.595986:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077248.595990:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617364480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077249.107283:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077249.491348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077249.491348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077249.491416:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077250.131370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077250.515386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077250.515392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077250.515460:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077250.643386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077250.643398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077250.643403:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077250.643414:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4239 00000100:00080000:0.0:1769077250.643425:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077250.643433:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077250.643478:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617365120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077250.643521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077250.643526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077250.643529:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077250.643535:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4239 00000100:00080000:0.0:1769077250.643540:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077250.643545:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077250.643557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077250.643564:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617365120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077250.643582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617365248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077250.643600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077250.643610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077250.643612:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077250.643617:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4239 00000100:00080000:0.0:1769077250.643631:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769077250.643637:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077250.643639:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077250.643650:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077250.643666:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617365376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077250.643679:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077250.643683:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077250.643685:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077250.643692:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077250/real 1769077250] req@ffff9b0365a3b100 x1855007617365120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077250.643694:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4239 00000100:00080000:0.0:1769077250.643717:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077250.643729:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077250.643735:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617365120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077250.643740:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077250.643766:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617365504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077250.643767:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077250.643786:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077250.643798:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617365120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077250.643802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077250.643809:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077250.643812:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077250.643819:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4239 00000100:00080000:0.0:1769077250.643824:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077250.643839:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077250.643840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617365248:192.168.204.160@tcp:38:kworker.0 00000008:00100000:0.0:1769077250.643859:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077250.643882:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617365632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077250.643897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077250.643924:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077250/real 1769077250] req@ffff9b0365a3bb80 x1855007617365248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077250.643942:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617365248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077250.643952:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077250.643956:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077250.643959:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617365248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077250.643968:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617365376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077250.643996:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077250.644008:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077250/real 1769077250] req@ffff9b0365a3ad80 x1855007617365376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077250.644019:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617365376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077250.644027:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077250.644030:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077250.644108:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617365376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077250.644119:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617365504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077250.644145:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077250.644155:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077250/real 1769077250] req@ffff9b0365a39180 x1855007617365504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077250.644167:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617365504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077250.644174:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077250.644178:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077250.644182:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617365504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077250.644199:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617365632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077250.644221:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077250.644257:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077250/real 1769077250] req@ffff9b0365a39500 x1855007617365632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077250.644272:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617365632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077250.644280:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077250.644283:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077250.644287:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617365632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077251.155353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077251.539381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077251.539421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077251.539461:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077252.179310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077252.563349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077252.563350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077252.563422:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077253.203320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077253.587348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077253.587348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077253.587414:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077254.227309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077254.611356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077254.611356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077254.611466:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077255.251320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077255.635338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077255.635339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077255.635411:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077255.763404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077255.763416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077255.763461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077255.763470:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4244 00000100:00080000:1.0:1769077255.763481:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077255.763488:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077255.763535:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617365760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077255.763557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077255.763560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077255.763562:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077255.763565:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4244 00000100:00080000:1.0:1769077255.763568:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077255.763571:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077255.763578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077255.763586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617365888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077255.763599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077255.763602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077255.763605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077255.763608:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4244 00000100:00080000:1.0:1769077255.763611:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077255.763613:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077255.763617:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077255.763628:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617366016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077255.763630:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617365760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077255.763652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077255.763655:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077255.763656:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077255.763659:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4244 00000100:00080000:1.0:1769077255.763661:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077255.763665:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077255.763668:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077255.763677:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617366144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077255.763694:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077255.763696:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077255.763698:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077255.763701:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4244 00000100:00080000:1.0:1769077255.763704:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077255.763707:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077255.763710:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077255.763716:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617366272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077255.763753:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077255.763810:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077255/real 1769077255] req@ffff9b0343deed80 x1855007617365760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077255.763833:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617365760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077255.763843:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077255.763847:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077255.763850:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617365760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077255.763857:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617365888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077255.763877:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077255.763886:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077255/real 1769077255] req@ffff9b0343deca80 x1855007617365888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077255.763895:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617365888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077255.763901:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077255.763904:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077255.763906:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617365888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077255.763933:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077255.763944:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077255.763951:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077255/real 1769077255] req@ffff9b0343deea00 x1855007617366016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077255.763975:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617366016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077255.763984:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077255.763987:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077255.763990:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077255.763995:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077255.764011:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077255.764019:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077255/real 1769077255] req@ffff9b0343def800 x1855007617366144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077255.764030:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617366144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077255.764036:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077255.764039:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077255.764042:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077255.764046:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077255.764059:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077255.764066:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077255/real 1769077255] req@ffff9b0343ded880 x1855007617366272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077255.764076:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617366272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077255.764090:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077255.764093:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077255.764096:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077256.275335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077256.659365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077256.659452:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077256.659462:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077257.299381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077257.683325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077257.683338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077257.683496:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077258.323290:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077258.707314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077258.707388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077258.707404:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077259.347352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077259.731339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077259.731339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077259.731429:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077260.371369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077260.755331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077260.755331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077260.755435:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077260.883399:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077260.883415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077260.883420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077260.883431:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4250 00000100:00080000:2.0:1769077260.883450:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077260.883457:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077260.883501:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617366400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077260.883529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077260.883532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077260.883534:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077260.883537:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4250 00000100:00080000:2.0:1769077260.883539:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077260.883542:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077260.883550:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077260.883560:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617366528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077260.883568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077260.883571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077260.883573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077260.883577:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4250 00000100:00080000:2.0:1769077260.883582:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077260.883591:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077260.883592:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617366400:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769077260.883596:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077260.883613:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617366656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077260.883625:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077260.883628:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077260.883630:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077260.883633:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4250 00000100:00080000:2.0:1769077260.883636:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077260.883638:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077260.883643:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077260.883652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617366784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077260.883660:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077260.883663:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077260.883664:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077260.883667:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4250 00000100:00080000:2.0:1769077260.883669:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769077260.883675:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077260.883680:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077260.883685:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077260.883700:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617366912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077260.883730:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077260/real 1769077260] req@ffff9b03482c4700 x1855007617366400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077260.883755:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617366400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077260.883765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077260.883768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077260.883771:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617366400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077260.883781:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077260.883808:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077260.883822:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077260/real 1769077260] req@ffff9b03482c6a00 x1855007617366528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077260.883837:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617366528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077260.883844:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077260.883847:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077260.883849:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077260.883870:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077260.883895:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077260.883903:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077260/real 1769077260] req@ffff9b03482c4000 x1855007617366656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077260.883911:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617366656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077260.883917:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077260.883919:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077260.883921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077260.883925:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077260.883937:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077260.883943:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077260/real 1769077260] req@ffff9b03482c7100 x1855007617366784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077260.883951:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617366784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077260.883956:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077260.883958:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077260.883961:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077260.883965:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077260.883975:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077260.883981:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077260/real 1769077260] req@ffff9b03482c6680 x1855007617366912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077260.883988:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617366912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077260.883993:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077260.883995:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077260.883997:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617366912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077261.395346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077261.779419:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077261.779434:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077261.779546:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077262.419412:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077262.803344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077262.803344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077262.803410:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077263.443358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077263.827309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077263.827339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077263.827363:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077264.467366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077264.851367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077264.851368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077264.851431:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077265.491395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077265.875372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077265.875435:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077265.875490:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077266.003366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.003375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077266.003380:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.003389:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4255 00000100:00080000:0.0:1769077266.003398:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.003404:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077266.003436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617367040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077266.003459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.003462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077266.003463:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.003467:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4255 00000100:00080000:0.0:1769077266.003469:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.003471:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077266.003478:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077266.003485:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617367168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077266.003492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.003495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077266.003496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.003499:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4255 00000100:00080000:0.0:1769077266.003501:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.003504:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077266.003508:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077266.003515:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617367296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077266.003519:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617367040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077266.003523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.003529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077266.003531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.003534:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4255 00000100:00080000:0.0:1769077266.003536:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.003538:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077266.003541:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077266.003549:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617367424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077266.003555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077266.003558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077266.003559:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077266.003561:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4255 00000100:00080000:0.0:1769077266.003563:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077266.003565:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077266.003568:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077266.003576:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617367552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077266.003596:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077266.003640:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9b0365a3aa00 x1855007617367040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077266.003656:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617367040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077266.003663:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077266.003667:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077266.003671:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617367040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077266.003680:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077266.003707:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077266.003720:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9b0365a3b480 x1855007617367168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077266.003734:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617367168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077266.003743:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077266.003746:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077266.003749:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077266.003773:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077266.003811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077266.003823:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9b0365a38e00 x1855007617367296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077266.003836:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617367296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077266.003844:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077266.003847:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077266.003851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077266.003857:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077266.003875:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077266.003887:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9b0365a39c00 x1855007617367424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077266.003899:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617367424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077266.003907:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077266.003910:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077266.003913:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077266.003920:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077266.003939:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077266.003950:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077266/real 1769077266] req@ffff9b0365a38a80 x1855007617367552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077266.003962:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617367552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077266.003970:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077266.003973:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077266.003976:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077266.515373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077266.899389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077266.899390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077266.899485:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077267.539356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077267.923350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077267.923404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077267.923496:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077268.563372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077268.947377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077268.947391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077268.947415:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077269.587321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077269.971370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077269.971371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077269.971482:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077270.611369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077270.995355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077270.995372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077270.995422:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077271.123423:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077271.123438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077271.123443:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077271.123458:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4260 00000100:00080000:0.0:1769077271.123473:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077271.123483:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077271.123547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617367680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077271.123577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077271.123580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077271.123582:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077271.123586:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4260 00000100:00080000:0.0:1769077271.123590:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077271.123592:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077271.123620:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077271.123632:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617367808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077271.123641:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077271.123644:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077271.123645:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077271.123648:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4260 00000100:00080000:0.0:1769077271.123651:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077271.123654:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077271.123657:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077271.123668:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617367936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077271.123669:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617367680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077271.123697:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077271.123700:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077271.123701:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077271.123704:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4260 00000100:00080000:0.0:1769077271.123707:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077271.123709:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077271.123714:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077271.123722:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617368064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077271.123730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077271.123733:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077271.123735:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077271.123738:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4260 00000100:00080000:0.0:1769077271.123741:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077271.123744:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077271.123747:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077271.123755:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617368192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077271.123790:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077271.123873:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077271/real 1769077271] req@ffff9b0365a38a80 x1855007617367680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077271.123896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617367680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077271.123924:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077271.123929:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077271.123933:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617367680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077271.123940:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077271.123963:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077271.123995:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077271/real 1769077271] req@ffff9b0365a39c00 x1855007617367808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077271.124010:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617367808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077271.124020:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077271.124024:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077271.124028:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077271.124084:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077271.124117:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077271.124125:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077271/real 1769077271] req@ffff9b0365a38e00 x1855007617367936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077271.124136:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617367936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077271.124154:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077271.124158:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077271.124199:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617367936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077271.124208:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077271.124287:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077271.124299:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077271/real 1769077271] req@ffff9b0365a3b480 x1855007617368064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077271.124309:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617368064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077271.124316:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077271.124319:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077271.124321:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077271.124326:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077271.124341:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077271.124348:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077271/real 1769077271] req@ffff9b0365a3aa00 x1855007617368192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077271.124357:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617368192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077271.124363:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077271.124365:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077271.124369:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077271.635307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077272.019360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077272.019360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077272.019517:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077272.659400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077273.043387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077273.043477:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077273.043521:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077273.683326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077274.067430:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077274.067438:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077274.067541:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077274.707376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077275.091382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077275.091387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077275.091538:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077275.731304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077276.115337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077276.115338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077276.115433:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077276.243431:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077276.243449:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077276.243457:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077276.243473:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4265 00000100:00080000:0.0:1769077276.243506:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077276.243517:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077276.243578:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617368320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077276.243611:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077276.243616:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077276.243635:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077276.243643:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4265 00000100:00080000:0.0:1769077276.243650:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077276.243658:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077276.243673:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077276.243717:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617368448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077276.243727:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617368320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077276.243756:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077276.243764:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077276.243790:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077276.243796:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4265 00000100:00080000:0.0:1769077276.243801:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077276.243807:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077276.243813:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077276.243828:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617368576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077276.243841:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769077276.243844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077276.243857:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077276.243860:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077276.243868:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4265 00000100:00080000:0.0:1769077276.243873:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077276.243878:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077276.243886:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077276.243892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077276/real 1769077276] req@ffff9b0365a39500 x1855007617368320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077276.243904:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617368704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077276.243920:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077276.243923:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617368320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077276.243927:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077276.243932:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077276.243943:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4265 00000100:00080000:3.0:1769077276.243945:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077276.243950:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077276.243956:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:0.0:1769077276.243960:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077276.243967:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617368320:192.168.204.160@tcp:250:kworker.0 00000008:00100000:0.0:1769077276.243970:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077276.243988:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617368832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077276.244020:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077276.244075:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077276.244145:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077276/real 1769077276] req@ffff9b0365a39180 x1855007617368448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077276.244174:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617368448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077276.244203:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077276.244211:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077276.244219:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077276.244290:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077276.244336:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077276.244358:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077276/real 1769077276] req@ffff9b0365a3ad80 x1855007617368576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077276.244383:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617368576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077276.244400:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077276.244406:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077276.244414:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077276.244426:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077276.244465:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077276.244491:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077276/real 1769077276] req@ffff9b0365a3bb80 x1855007617368704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077276.244515:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617368704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077276.244530:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077276.244536:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077276.244543:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077276.244580:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077276.244603:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077276.244616:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077276/real 1769077276] req@ffff9b0365a3b100 x1855007617368832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077276.244649:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617368832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077276.244670:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077276.244677:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077276.244683:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617368832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077276.755376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077277.139332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077277.139333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077277.139434:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077277.779311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077278.163337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077278.163337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077278.163442:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077278.803431:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077279.187347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077279.187348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077279.187403:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077279.827356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077280.211418:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077280.211494:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077280.211518:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077280.851341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077281.235304:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077281.235304:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077281.235360:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077281.363400:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077281.363430:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077281.363435:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077281.363448:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4270 00000100:00080000:0.0:1769077281.363462:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077281.363472:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077281.363513:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617368960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077281.363544:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077281.363549:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077281.363551:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077281.363555:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4270 00000100:00080000:0.0:1769077281.363559:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077281.363564:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077281.363574:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077281.363589:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617369088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077281.363600:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077281.363621:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617368960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077281.363622:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077281.363625:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077281.363633:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4270 00000100:00080000:0.0:1769077281.363636:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077281.363640:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077281.363644:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077281.363658:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617369216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077281.363666:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077281.363668:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077281.363670:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077281.363673:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4270 00000100:00080000:0.0:1769077281.363678:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077281.363681:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077281.363685:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077281.363695:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617369344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077281.363703:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769077281.363706:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077281.363708:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077281.363712:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077281.363717:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4270 00000100:00080000:0.0:1769077281.363721:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077281.363724:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077281.363729:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077281.363739:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617369472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077281.363816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077281/real 1769077281] req@ffff9b0365a3b100 x1855007617368960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077281.363833:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617368960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077281.363842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077281.363846:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077281.363850:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617368960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077281.363870:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077281.363890:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077281.363900:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077281/real 1769077281] req@ffff9b0365a3bb80 x1855007617369088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077281.363910:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617369088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077281.363916:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077281.363919:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077281.363921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077281.363926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077281.363938:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077281.363944:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077281/real 1769077281] req@ffff9b0365a3ad80 x1855007617369216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077281.363965:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617369216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077281.363970:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077281.363972:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077281.363974:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077281.363979:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077281.364015:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077281.364047:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077281/real 1769077281] req@ffff9b0365a39180 x1855007617369344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077281.364064:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617369344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077281.364075:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077281.364079:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077281.364083:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077281.364090:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077281.364114:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077281.364148:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077281/real 1769077281] req@ffff9b0365a39500 x1855007617369472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077281.364157:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617369472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077281.364163:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077281.364165:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077281.364168:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077281.875408:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077282.259345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077282.259362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077282.259446:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077282.899403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077283.283295:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077283.283313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077283.283363:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077283.923349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077284.307323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077284.307323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077284.307423:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077284.947354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077285.331294:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077285.331305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077285.331385:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077285.971350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077286.355346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077286.355346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077286.355378:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077286.483355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077286.483363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077286.483367:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077286.483375:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4275 00000100:00080000:2.0:1769077286.483383:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077286.483390:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077286.483428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617369600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077286.483453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077286.483456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077286.483458:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077286.483461:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4275 00000100:00080000:2.0:1769077286.483465:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077286.483468:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077286.483476:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077286.483485:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617369600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077286.483489:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617369728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077286.483499:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077286.483501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077286.483503:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077286.483506:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4275 00000100:00080000:2.0:1769077286.483508:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077286.483511:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077286.483514:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077286.483522:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617369856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077286.483528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077286.483530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077286.483531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077286.483533:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4275 00000100:00080000:2.0:1769077286.483537:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077286.483539:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077286.483543:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077286.483548:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077286.483553:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617369984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077286.483561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077286.483564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077286.483565:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077286.483568:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4275 00000100:00080000:2.0:1769077286.483570:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077286.483572:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077286.483575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077286.483584:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617370112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077286.483587:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077286/real 1769077286] req@ffff9b03482c6d80 x1855007617369600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077286.483614:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617369600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077286.483623:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077286.483627:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077286.483630:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617369600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077286.483649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077286.483665:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077286.483673:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077286/real 1769077286] req@ffff9b03482c6300 x1855007617369728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077286.483681:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617369728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077286.483687:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077286.483689:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077286.483691:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077286.483695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077286.483705:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077286.483711:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077286/real 1769077286] req@ffff9b03482c5500 x1855007617369856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077286.483718:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617369856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077286.483723:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077286.483725:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077286.483727:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077286.483731:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077286.483741:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077286.483746:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077286/real 1769077286] req@ffff9b03482c4a80 x1855007617369984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077286.483753:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617369984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077286.483758:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077286.483760:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077286.483762:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617369984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077286.483766:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617370112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077286.483774:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077286.483780:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077286/real 1769077286] req@ffff9b03482c5180 x1855007617370112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077286.483787:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617370112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077286.483792:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077286.483793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077286.483795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617370112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077286.995319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077287.379404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077287.379523:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077287.379529:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077288.019395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077288.403368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077288.403369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077288.403438:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077289.043352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077289.427384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077289.427384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077289.427492:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077290.067349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077290.451360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077290.451361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077290.451465:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077291.091331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077291.475402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077291.475402:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077291.475454:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077291.603456:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077291.603468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077291.603473:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077291.603484:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4280 00000100:00080000:0.0:1769077291.603496:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077291.603503:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077291.603547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617370240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077291.603593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077291.603596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077291.603598:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077291.603601:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4280 00000100:00080000:0.0:1769077291.603603:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077291.603606:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077291.603634:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077291.603663:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617370368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077291.603676:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077291.603699:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617370240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077291.603701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077291.603704:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077291.603723:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4280 00000100:00080000:0.0:1769077291.603728:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077291.603733:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077291.603739:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077291.603753:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617370496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077291.603765:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077291.603768:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077291.603771:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077291.603776:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4280 00000100:00080000:0.0:1769077291.603783:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769077291.603791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077291.603796:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077291.603805:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077291.603852:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617370624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077291.603872:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077291.603875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077291.603877:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077291.603880:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4280 00000100:00080000:0.0:1769077291.603884:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077291.603887:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077291/real 1769077291] req@ffff9b0365a3aa00 x1855007617370240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077291.603892:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077291.603903:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077291.603929:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617370752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077291.603935:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617370240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077291.603973:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077291.603978:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077291.603983:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617370240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077291.604005:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617370368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077291.604025:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077291.604033:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077291/real 1769077291] req@ffff9b0365a3b480 x1855007617370368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077291.604041:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617370368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077291.604047:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077291.604049:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077291.604052:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617370368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077291.604058:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617370496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077291.604078:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077291.604089:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077291/real 1769077291] req@ffff9b0365a38e00 x1855007617370496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077291.604102:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617370496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077291.604111:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077291.604115:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077291.604118:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617370496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077291.604125:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617370624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077291.604146:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077291.604154:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077291/real 1769077291] req@ffff9b0365a39c00 x1855007617370624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077291.604163:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617370624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077291.604168:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077291.604171:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077291.604173:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617370624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077291.604176:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617370752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077291.604188:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077291.604221:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077291/real 1769077291] req@ffff9b0365a38a80 x1855007617370752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077291.604230:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617370752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077291.604242:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077291.604244:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077291.604246:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617370752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077292.115373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077292.500050:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077292.500176:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077292.500894:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077293.139328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077293.523419:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077293.523429:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077293.523558:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077294.163369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077294.547431:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077294.547548:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077294.548848:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077295.187314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769077295.571515:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077295.571714:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077295.571715:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077296.211349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077296.595764:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077296.595842:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077296.595856:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077296.723325:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077296.723334:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077296.723338:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077296.723346:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4285 00000100:00080000:0.0:1769077296.723356:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077296.723405:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077296.723440:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617370880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077296.723464:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077296.723466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077296.723468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077296.723471:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4285 00000100:00080000:0.0:1769077296.723474:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077296.723476:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077296.723483:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077296.723491:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617371008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077296.723497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077296.723500:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077296.723501:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077296.723503:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4285 00000100:00080000:0.0:1769077296.723505:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077296.723507:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077296.723510:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077296.723516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617371136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077296.723522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077296.723524:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077296.723525:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077296.723528:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4285 00000100:00080000:0.0:1769077296.723530:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077296.723533:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077296.723537:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077296.723542:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617370880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077296.723545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617371264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077296.723555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077296.723557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077296.723558:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077296.723561:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4285 00000100:00080000:0.0:1769077296.723564:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077296.723566:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077296.723569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077296.723577:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617371392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077296.723607:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077296.723641:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077296/real 1769077296] req@ffff9b0365a38a80 x1855007617370880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077296.723656:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617370880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077296.723664:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077296.723667:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077296.723670:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617370880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077296.723675:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077296.723688:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077296.723695:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077296/real 1769077296] req@ffff9b0365a39c00 x1855007617371008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077296.723713:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617371008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077296.723722:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077296.723726:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077296.723730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077296.723750:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077296.723773:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077296.723784:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077296/real 1769077296] req@ffff9b0365a38e00 x1855007617371136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077296.723793:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617371136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077296.723799:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077296.723801:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077296.723804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077296.723825:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077296.723839:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077296.723850:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077296/real 1769077296] req@ffff9b0365a3b480 x1855007617371264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077296.723859:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617371264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077296.723869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077296.723875:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077296.723878:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077296.723882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077296.723892:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077296.723897:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077296/real 1769077296] req@ffff9b0365a3aa00 x1855007617371392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077296.723905:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617371392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077296.723910:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077296.723915:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077296.723917:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077297.235327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077297.619379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077297.619643:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077297.619710:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077298.259356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077298.643343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077298.643375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077298.643396:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077299.283358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077299.667358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077299.667359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077299.667397:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077300.307340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077300.691326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077300.691366:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077300.691368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077301.331352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769077301.715364:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077301.715364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077301.715421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:0.0:1769077301.843332:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077301.843343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077301.843348:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077301.843359:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4291 00000100:00080000:0.0:1769077301.843368:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077301.844553:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077301.844589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617371520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077301.844626:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077301.844629:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077301.844632:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077301.844637:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4291 00000100:00080000:0.0:1769077301.844641:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077301.844660:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077301.844670:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077301.844687:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617371648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077301.844699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077301.844701:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617371520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077301.844704:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077301.844709:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077301.844715:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4291 00000100:00080000:0.0:1769077301.844719:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077301.844723:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077301.844729:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077301.844740:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617371776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077301.844751:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077301.844755:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077301.844757:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077301.844761:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4291 00000100:00080000:0.0:1769077301.844766:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077301.844771:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077301.844776:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769077301.844778:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077301.844790:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617371904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077301.844802:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077301.844806:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077301.844808:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077301.844813:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4291 00000100:00080000:0.0:1769077301.844817:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077301.844820:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077301.844826:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077301.844838:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077301/real 1769077301] req@ffff9b0365a39500 x1855007617371520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077301.844840:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617372032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077301.844909:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617371520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077301.844920:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077301.844926:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077301.844930:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617371520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077301.844952:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077301.844999:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077301.845012:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077301/real 1769077301] req@ffff9b0365a39180 x1855007617371648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077301.845025:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617371648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077301.845034:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077301.845038:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077301.845042:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077301.845048:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077301.845065:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077301.845075:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077301/real 1769077301] req@ffff9b0365a3ad80 x1855007617371776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077301.845088:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617371776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077301.845101:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077301.845104:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077301.845108:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077301.845114:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077301.845130:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077301.845140:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077301/real 1769077301] req@ffff9b0365a3bb80 x1855007617371904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077301.845151:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617371904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077301.845158:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077301.845161:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077301.845164:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617371904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077301.845170:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077301.845187:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077301.845196:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077301/real 1769077301] req@ffff9b0365a3b100 x1855007617372032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077301.845207:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617372032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077301.845214:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077301.845217:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077301.845220:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077302.355347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077302.739303:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077302.739311:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077302.739370:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077303.379375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077303.763310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077303.763312:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077303.763425:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077304.403326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077304.787313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077304.787313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077304.787360:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077305.427413:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077305.811380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077305.811380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077305.811466:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077306.451393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077306.835299:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077306.835377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077306.835405:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077306.963335:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077306.963346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077306.963351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077306.963362:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4296 00000100:00080000:0.0:1769077306.963373:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077306.963380:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077306.963416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617372160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077306.963441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077306.963445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077306.963446:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077306.963449:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4296 00000100:00080000:0.0:1769077306.963452:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077306.963455:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077306.963463:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077306.963471:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617372288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077306.963479:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077306.963482:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077306.963484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077306.963489:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617372160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077306.963490:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4296 00000100:00080000:0.0:1769077306.963495:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077306.963500:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077306.963503:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077306.963510:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617372416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077306.963516:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077306.963519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077306.963520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077306.963523:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4296 00000100:00080000:0.0:1769077306.963525:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077306.963528:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077306.963532:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077306.963539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617372544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077306.963545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077306.963548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077306.963549:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077306.963554:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4296 00000800:00000100:3.0:1769077306.963557:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077306.963559:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077306.963577:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077306.963582:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077306.963592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617372672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077306.963618:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077306/real 1769077306] req@ffff9b0365a3b100 x1855007617372160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077306.963640:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617372160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077306.963649:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077306.963652:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077306.963655:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617372160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077306.963679:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077306.963705:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077306.963719:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077306/real 1769077306] req@ffff9b0365a3bb80 x1855007617372288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077306.963732:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617372288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077306.963739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077306.963741:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077306.963744:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077306.963750:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077306.963772:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077306.963785:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077306/real 1769077306] req@ffff9b0365a3ad80 x1855007617372416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077306.963799:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617372416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077306.963808:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077306.963811:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077306.963815:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077306.963822:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077306.963844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077306.963855:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077306/real 1769077306] req@ffff9b0365a39180 x1855007617372544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077306.963868:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617372544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077306.963885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077306.963888:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077306.963892:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077306.963899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077306.963919:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077306.963930:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077306/real 1769077306] req@ffff9b0365a39500 x1855007617372672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077306.963942:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617372672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077306.963950:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077306.963953:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077306.963956:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077307.475424:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077307.859313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077307.859327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077307.860585:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077308.499338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077308.883377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077308.883560:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077308.883569:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077309.523344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077309.907391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077309.907392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077309.907470:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077310.547344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077310.931342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077310.931342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077310.931397:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077311.571353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077311.955332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077311.955332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077311.955348:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077312.083361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077312.083369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077312.083373:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077312.083379:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4301 00000100:00080000:3.0:1769077312.083387:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077312.083392:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077312.083428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617372800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077312.083457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077312.083461:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077312.083462:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077312.083467:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4301 00000100:00080000:3.0:1769077312.083470:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077312.083488:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077312.083495:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077312.083506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617372928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077312.083514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077312.083517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077312.083518:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077312.083522:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4301 00000100:00100000:1.0:1769077312.083524:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617372800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077312.083525:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077312.083530:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077312.083535:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077312.083541:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617373056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077312.083547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077312.083549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077312.083550:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077312.083552:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4301 00000100:00080000:3.0:1769077312.083554:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077312.083555:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077312.083558:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077312.083566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617373184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077312.083571:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077312.083573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077312.083574:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077312.083576:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4301 00000100:00080000:3.0:1769077312.083579:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077312.083581:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077312.083583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077312.083590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617373312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077312.083629:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077312.083661:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077312.085149:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077312.085177:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077312.595308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077312.979356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077312.979356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077312.979407:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077313.619334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077314.003388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077314.003388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077314.003567:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077314.643343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077315.027345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077315.027345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077315.027394:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769077315.155531:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769077315.155564:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769077315.155582:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077315.155627:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077315.155636:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077315.155645:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077315.155654:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769077315.155668:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077312/real 1769077315] req@ffff9b0352a1ad80 x1855007617372928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077315.155693:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617372928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077315.155705:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077315.155710:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077315.155714:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617372928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077315.155721:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077312/real 1769077315] req@ffff9b0352a1b480 x1855007617373056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077315.155733:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617373056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077315.155740:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077315.155743:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077315.155746:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077315.155753:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077312/real 1769077315] req@ffff9b0352a18a80 x1855007617373184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077315.155764:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617373184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077315.155771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077315.155774:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077315.155778:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077315.155784:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077312/real 1769077315] req@ffff9b0352a1a680 x1855007617373312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077315.155794:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617373312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077315.155801:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077315.155804:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077315.155808:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077315.155814:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077312/real 1769077315] req@ffff9b0352a1b100 x1855007617372800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077315.155824:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617372800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077315.155831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077315.155834:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077315.155837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617372800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077315.667302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769077316.051449:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077316.051463:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077316.051549:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077316.691375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077317.075335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077317.075344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077317.075431:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077317.203316:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077317.203337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077317.203342:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077317.203355:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4306 00000100:00080000:3.0:1769077317.203368:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077317.203376:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077317.203428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617373440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077317.203458:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077317.203465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077317.203467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077317.203471:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4306 00000100:00080000:3.0:1769077317.203476:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077317.203481:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077317.203492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077317.203504:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617373440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077317.203506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617373568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077317.203519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077317.203523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077317.203525:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077317.203530:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4306 00000100:00080000:3.0:1769077317.203533:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077317.203537:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077317.203541:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077317.203552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617373696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077317.203561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077317.203565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077317.203566:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077317.203570:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4306 00000100:00080000:3.0:1769077317.203574:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077317.203579:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077317.203583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077317.203589:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077317.203599:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617373824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077317.203614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077317.203618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077317.203620:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077317.203623:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4306 00000100:00080000:3.0:1769077317.203627:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077317.203633:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077317/real 1769077317] req@ffff9b0352a18700 x1855007617373440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077317.203633:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077317.203640:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077317.203653:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617373952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077317.203669:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617373440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077317.203681:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077317.203686:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077317.203691:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617373440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077317.203720:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077317.203747:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077317.203761:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077317/real 1769077317] req@ffff9b0352a1aa00 x1855007617373568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077317.203772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617373568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077317.203781:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077317.203784:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077317.203787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077317.203793:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077317.203813:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077317.203824:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077317/real 1769077317] req@ffff9b0352a18e00 x1855007617373696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077317.203835:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617373696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077317.203848:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077317.203852:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077317.203855:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077317.203860:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077317.203882:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077317.203894:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077317/real 1769077317] req@ffff9b0352a1a300 x1855007617373824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077317.203906:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617373824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077317.203914:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077317.203917:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077317.203928:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077317.203935:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077317.203954:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077317.203964:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077317/real 1769077317] req@ffff9b0352a19f80 x1855007617373952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077317.203974:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617373952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077317.203982:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077317.203984:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077317.203988:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617373952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077317.715366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077318.099360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077318.099426:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077318.099432:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077318.739357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077319.123379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077319.123386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077319.123514:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077319.763317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077320.147312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077320.147363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077320.147424:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077320.787362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077321.171343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077321.171366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077321.171430:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077321.811355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077322.195339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077322.195361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077322.195423:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077322.323474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077322.323484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077322.323489:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077322.323496:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4311 00000100:00080000:2.0:1769077322.323505:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077322.323511:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077322.323542:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617374080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077322.323563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077322.323566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077322.323567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077322.323570:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4311 00000100:00080000:2.0:1769077322.323572:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077322.323574:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077322.323581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077322.323593:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617374208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077322.323606:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077322.323609:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077322.323610:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077322.323616:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4311 00000100:00100000:1.0:1769077322.323620:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617374080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077322.323621:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077322.323633:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077322.323638:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077322.323651:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617374336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077322.323658:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077322.323661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077322.323662:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077322.323664:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4311 00000100:00080000:2.0:1769077322.323666:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077322.323668:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077322.323672:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077322.323681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617374464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077322.323697:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077322.323700:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077322.323702:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077322.323705:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4311 00000100:00080000:2.0:1769077322.323710:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077322.323714:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077322.323719:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769077322.323725:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077322.323736:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617374592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077322.323782:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077322/real 1769077322] req@ffff9b03482c4380 x1855007617374080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077322.323800:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617374080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077322.323809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077322.323813:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077322.323816:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617374080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077322.323822:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077322.323842:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077322.323850:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077322/real 1769077322] req@ffff9b03482c5180 x1855007617374208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077322.323857:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617374208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077322.323863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077322.323866:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077322.323868:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077322.323891:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077322.323901:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077322.323907:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077322/real 1769077322] req@ffff9b03482c4a80 x1855007617374336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077322.323913:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617374336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077322.323919:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077322.323920:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077322.323922:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077322.323926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077322.323936:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077322.323941:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077322/real 1769077322] req@ffff9b03482c5500 x1855007617374464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077322.323947:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617374464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077322.323952:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077322.323953:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077322.323956:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077322.323960:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077322.323969:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077322.323974:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077322/real 1769077322] req@ffff9b03482c6300 x1855007617374592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077322.323980:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617374592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077322.323985:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077322.323987:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077322.323988:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077322.835385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077323.219364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077323.219423:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077323.219473:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077323.859384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077324.243315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077324.243340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077324.243428:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077324.883400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077325.267378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077325.267379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077325.267450:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077325.907356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077326.291365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077326.291365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077326.291392:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077326.931363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077327.315354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077327.315354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077327.315373:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077327.443381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077327.443392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077327.443396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077327.443403:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4316 00000100:00080000:3.0:1769077327.443412:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077327.443416:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077327.443446:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617374720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077327.443490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077327.443495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077327.443496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077327.443501:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4316 00000100:00080000:3.0:1769077327.443504:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077327.443508:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077327.443518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077327.443528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617374848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077327.443535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077327.443537:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617374720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077327.443538:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077327.443542:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077327.443546:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4316 00000100:00080000:3.0:1769077327.443549:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077327.443551:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077327.443554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077327.443562:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617374976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077327.443568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077327.443570:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077327.443572:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077327.443574:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4316 00000100:00080000:3.0:1769077327.443577:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077327.443579:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077327.443582:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077327.443587:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617375104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077327.443593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077327.443596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077327.443598:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077327.443602:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4316 00000800:00000100:1.0:1769077327.443605:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077327.443607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077327.443613:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077327.443618:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077327.443630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617375232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077327.443649:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077327/real 1769077327] req@ffff9b0352a19f80 x1855007617374720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077327.443672:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617374720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077327.443679:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077327.443682:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077327.443685:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617374720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077327.443707:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077327.443732:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077327.443743:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077327/real 1769077327] req@ffff9b0352a1a300 x1855007617374848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077327.443753:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617374848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077327.443761:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077327.443764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077327.443767:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077327.443772:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077327.443789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077327.443797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077327/real 1769077327] req@ffff9b0352a18e00 x1855007617374976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077327.443806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617374976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077327.443818:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077327.443820:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077327.443823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617374976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077327.443828:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077327.443844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077327.443853:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077327/real 1769077327] req@ffff9b0352a1aa00 x1855007617375104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077327.443863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617375104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077327.443870:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077327.443873:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077327.443876:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077327.443879:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077327.443892:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077327.443898:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077327/real 1769077327] req@ffff9b0352a18700 x1855007617375232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077327.443905:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617375232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077327.443909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077327.443911:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077327.443913:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077327.955333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077328.339341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077328.339341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077328.339414:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077328.979326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077329.363332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077329.363332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077329.363388:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077330.003301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077330.387336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077330.387336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077330.387390:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077331.027335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077331.411309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077331.411309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077331.411331:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077332.051325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077332.435337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077332.435337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077332.435366:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077332.563320:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077332.563329:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077332.563332:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077332.563341:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4321 00000100:00080000:3.0:1769077332.563349:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077332.563354:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077332.563385:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617375360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077332.563409:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077332.563412:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077332.563414:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077332.563417:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4321 00000100:00080000:3.0:1769077332.563419:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077332.563421:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077332.563429:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077332.563452:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617375488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077332.563456:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617375360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077332.563474:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077332.563479:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077332.563481:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077332.563487:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4321 00000100:00080000:3.0:1769077332.563492:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077332.563496:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077332.563500:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077332.563512:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617375616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077332.563518:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077332.563530:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077332.563534:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077332.563537:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077332.563543:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4321 00000100:00080000:3.0:1769077332.563548:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077332.563552:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077332.563559:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077332.563567:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077332/real 1769077332] req@ffff9b0352a1b100 x1855007617375360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077332.563575:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617375744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077332.563584:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077332.563588:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617375360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077332.563589:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077332.563593:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077332.563601:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4321 00000100:00080000:3.0:1769077332.563606:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077332.563607:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077332.563612:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:1.0:1769077332.563615:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000008:00100000:3.0:1769077332.563618:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077332.563621:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617375360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077332.563630:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617375872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077332.563646:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077332.563665:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077332.563675:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077332/real 1769077332] req@ffff9b0352a1a680 x1855007617375488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077332.563683:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617375488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077332.563689:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077332.563691:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077332.563694:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077332.563697:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077332.563707:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077332.563713:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077332/real 1769077332] req@ffff9b0352a18a80 x1855007617375616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077332.563721:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617375616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077332.563726:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077332.563727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077332.563729:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077332.563733:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077332.563742:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077332.563747:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077332/real 1769077332] req@ffff9b0352a1b480 x1855007617375744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077332.563754:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617375744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077332.563759:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077332.563761:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077332.563764:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077332.563767:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077332.563776:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077332.563782:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077332/real 1769077332] req@ffff9b0352a1ad80 x1855007617375872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077332.563789:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617375872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077332.563794:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077332.563795:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077332.563797:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617375872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077333.075337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077333.459356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077333.459435:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077333.459476:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077334.099300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077334.483340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077334.483340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077334.483407:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077335.123368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077335.507315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077335.507315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077335.507375:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077336.147300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077336.531381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077336.531382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077336.531400:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077337.171353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077337.555329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077337.555329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077337.555360:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077337.683366:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077337.683379:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077337.683384:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077337.683395:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4326 00000100:00080000:3.0:1769077337.683407:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077337.683415:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077337.683458:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617376000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077337.683487:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077337.683490:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077337.683491:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077337.683495:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4326 00000100:00080000:3.0:1769077337.683497:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077337.683500:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077337.683508:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077337.683518:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617376128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077337.683526:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077337.683529:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077337.683530:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077337.683533:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4326 00000100:00080000:3.0:1769077337.683536:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077337.683539:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077337.683542:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077337.683548:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617376256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077337.683557:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077337.683561:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077337.683565:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077337.683569:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617376000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077337.683570:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4326 00000100:00080000:3.0:1769077337.683578:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077337.683585:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077337.683589:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077337.683600:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617376384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077337.683606:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077337.683608:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077337.683610:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077337.683614:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4326 00000100:00080000:3.0:1769077337.683620:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077337.683623:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077337.683628:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077337.683638:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617376512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077337.683647:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077337.683699:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077337/real 1769077337] req@ffff9b0352a18000 x1855007617376000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077337.683727:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617376000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077337.683739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077337.683744:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077337.683748:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617376000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077337.683755:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077337.683774:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077337.683783:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077337/real 1769077337] req@ffff9b0352a18380 x1855007617376128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077337.683791:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617376128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077337.683797:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077337.683799:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077337.683801:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077337.683822:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077337.683833:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077337.683839:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077337/real 1769077337] req@ffff9b0352a19c00 x1855007617376256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077337.683846:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617376256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077337.683855:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077337.683858:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077337.683860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077337.683864:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077337.683883:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077337.683890:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077337/real 1769077337] req@ffff9b0352a1b800 x1855007617376384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077337.683897:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617376384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077337.683902:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077337.683905:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077337.683907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077337.683910:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077337.683921:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077337.683926:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077337/real 1769077337] req@ffff9b0352a19500 x1855007617376512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077337.683933:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617376512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077337.683938:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077337.683940:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077337.683943:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077338.195366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077338.579381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077338.579381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077338.579456:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077339.219336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077339.603354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077339.603366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077339.603426:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077340.243360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077340.627388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077340.627389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077340.627459:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077341.267355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077341.651370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077341.651371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077341.651390:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077342.291364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077342.675370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077342.675396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077342.675434:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077342.803360:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077342.803369:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077342.803374:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077342.803383:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4331 00000100:00080000:3.0:1769077342.803394:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077342.803402:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077342.803440:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617376640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077342.803476:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077342.803480:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077342.803481:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077342.803485:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4331 00000100:00080000:3.0:1769077342.803488:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077342.803491:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077342.803499:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077342.803511:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617376768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077342.803541:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077342.803546:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077342.803548:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077342.803554:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4331 00000100:00080000:3.0:1769077342.803558:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077342.803566:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617376640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077342.803568:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077342.803582:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077342.803596:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617376896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077342.803610:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077342.803614:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077342.803616:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077342.803620:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4331 00000100:00080000:3.0:1769077342.803626:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077342.803630:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077342.803637:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077342.803654:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617377024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077342.803672:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077342.803681:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1769077342.803686:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077342.803688:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077342.803703:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4331 00000100:00080000:3.0:1769077342.803712:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077342.803720:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077342.803732:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077342.803755:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617377152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077342.803768:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077342/real 1769077342] req@ffff9b0352a19500 x1855007617376640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077342.803811:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617376640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077342.803822:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077342.803827:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077342.803831:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617376640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077342.803838:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077342.803860:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077342.803871:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077342/real 1769077342] req@ffff9b0352a1b800 x1855007617376768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077342.803887:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617376768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077342.803895:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077342.803897:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077342.803901:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077342.803923:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077342.803936:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077342.803944:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077342/real 1769077342] req@ffff9b0352a19c00 x1855007617376896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077342.803954:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617376896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077342.803994:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077342.803997:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077342.804000:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617376896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077342.804006:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077342.804022:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077342.804031:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077342/real 1769077342] req@ffff9b0352a18380 x1855007617377024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077342.804041:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617377024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077342.804053:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077342.804056:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077342.804058:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077342.804063:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077342.804075:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077342.804081:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077342/real 1769077342] req@ffff9b0352a18000 x1855007617377152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077342.804091:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617377152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077342.804098:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077342.804100:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077342.804102:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077343.315349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077343.699356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077343.699356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077343.699408:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077344.339366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077344.723382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077344.723383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077344.723440:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077345.363333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077345.747321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077345.747322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077345.747410:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077346.387365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077346.771315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077346.771334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077346.771375:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077347.411364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077347.795345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077347.795374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077347.795400:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077347.923436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077347.923448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077347.923453:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077347.923463:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4337 00000100:00080000:0.0:1769077347.923474:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077347.923481:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077347.923516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617377280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077347.923551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077347.923555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077347.923557:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077347.923561:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4337 00000100:00080000:0.0:1769077347.923565:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077347.923569:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077347.923579:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077347.923593:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617377408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077347.923600:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617377280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077347.923613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077347.923617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077347.923620:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077347.923625:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4337 00000100:00080000:0.0:1769077347.923629:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077347.923633:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077347.923638:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077347.923650:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617377536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077347.923661:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1769077347.923664:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077347.923668:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077347.923672:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077347.923677:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4337 00000100:00080000:0.0:1769077347.923682:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077347.923687:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077347.923692:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077347.923703:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617377664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077347.923714:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077347/real 1769077347] req@ffff9b0365a3aa00 x1855007617377280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077347.923716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077347.923725:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077347.923728:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077347.923737:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4337 00000100:00080000:0.0:1769077347.923742:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077347.923746:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077347.923749:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617377280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077347.923753:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769077347.923764:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077347.923770:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077347.923771:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617377792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077347.923775:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617377280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077347.923802:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077347.923822:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077347.923833:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077347/real 1769077347] req@ffff9b0365a3b480 x1855007617377408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077347.923842:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617377408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077347.923848:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077347.923850:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077347.923853:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077347.923860:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077347.923891:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077347.923904:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077347/real 1769077347] req@ffff9b0365a38e00 x1855007617377536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077347.923925:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617377536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077347.923932:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077347.923934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077347.923937:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077347.923941:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077347.923957:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077347.923964:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077347/real 1769077347] req@ffff9b0365a39c00 x1855007617377664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077347.923971:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617377664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077347.923976:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077347.923978:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077347.923980:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077347.923983:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077347.923993:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077347.923998:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077347/real 1769077347] req@ffff9b0365a38a80 x1855007617377792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077347.924005:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617377792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077347.924014:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077347.924016:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077347.924018:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617377792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077348.435415:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077348.819403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077348.819404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077348.819489:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077349.459343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077349.843348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077349.843348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077349.843455:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077350.483357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077350.867350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077350.867350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077350.867405:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077351.507317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077351.891342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077351.891342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077351.891379:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077352.531334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077352.915338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077352.915342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077352.915357:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077353.043427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.043440:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077353.043446:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.043457:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4342 00000100:00080000:3.0:1769077353.043471:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077353.043481:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077353.043543:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617377920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077353.043565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.043569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077353.043571:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.043578:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4342 00000100:00080000:3.0:1769077353.043583:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077353.043589:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077353.043601:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077353.043631:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617377920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077353.043634:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617378048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077353.043656:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.043661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077353.043664:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.043671:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4342 00000100:00080000:3.0:1769077353.043677:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077353.043728:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077353.043769:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff9b0352a1ad80 x1855007617377920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077353.043770:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077353.043781:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077353.043830:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617377920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077353.043840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077353.043844:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077353.043849:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617377920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077353.043883:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077353.043903:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077353.043915:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff9b0352a1b480 x1855007617378048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077353.043925:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617378048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077353.043931:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077353.043934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077353.043936:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077353.043985:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617378176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077353.044001:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.044004:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077353.044006:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.044011:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4342 00000100:00100000:1.0:1769077353.044034:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077353.044059:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077353.044071:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077353.044077:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff9b0352a18a80 x1855007617378176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077353.044081:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077353.044113:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077353.044115:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617378176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077353.044130:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077353.044136:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617378304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077353.044137:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077353.044151:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378176:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769077353.044184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077353.044187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077353.044190:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077353.044195:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4342 00000100:00080000:3.0:1769077353.044199:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077353.044201:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378304:192.168.204.160@tcp:8:kworker.0 00000100:00080000:3.0:1769077353.044212:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077353.044219:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769077353.044263:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077353.044279:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff9b0352a1a680 x1855007617378304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077353.044289:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617378304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077353.044295:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617378432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077353.044308:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077353.044322:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077353.044330:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077353.044346:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077353.044364:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077353.044371:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077353/real 1769077353] req@ffff9b0352a1b100 x1855007617378432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077353.044378:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617378432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077353.044383:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077353.044385:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077353.044387:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077353.555320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077353.939357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077353.939357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077353.939469:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077354.579382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077354.963372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077354.963375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077354.963479:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077355.603455:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077355.987376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077355.987376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077355.987513:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077356.627384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077357.011390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077357.011390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077357.011460:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077357.651337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077358.035357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077358.035357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077358.035391:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077358.163322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077358.163333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077358.163350:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077358.163360:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4347 00000100:00080000:3.0:1769077358.163368:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077358.163374:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077358.163407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617378560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077358.163432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077358.163435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077358.163437:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077358.163440:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4347 00000100:00080000:3.0:1769077358.163442:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077358.163445:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077358.163452:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077358.163460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617378688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077358.163466:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077358.163469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077358.163470:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077358.163473:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4347 00000100:00080000:3.0:1769077358.163474:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077358.163476:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077358.163479:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077358.163485:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617378816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077358.163490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077358.163492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077358.163493:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077358.163496:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4347 00000100:00080000:3.0:1769077358.163499:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077358.163501:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077358.163505:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077358.163510:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617378560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077358.163514:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617378944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077358.163522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077358.163525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077358.163526:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077358.163536:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4347 00000100:00080000:3.0:1769077358.163539:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077358.163541:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077358.163545:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077358.163552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617379072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077358.163568:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077358.163603:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077358/real 1769077358] req@ffff9b0352a18700 x1855007617378560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077358.163641:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617378560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077358.163650:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077358.163654:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077358.163657:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617378560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077358.163663:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077358.163679:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077358.163687:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077358/real 1769077358] req@ffff9b0352a1aa00 x1855007617378688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077358.163696:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617378688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077358.163738:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077358.163741:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077358.163743:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077358.163747:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077358.163764:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077358.163772:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077358/real 1769077358] req@ffff9b0352a18e00 x1855007617378816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077358.163780:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617378816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077358.163789:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077358.163791:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077358.163793:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077358.163812:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077358.163823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077358.163829:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077358/real 1769077358] req@ffff9b0352a1a300 x1855007617378944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077358.163836:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617378944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077358.163841:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077358.163854:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077358.163856:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617378944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077358.163859:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077358.163868:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077358.163873:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077358/real 1769077358] req@ffff9b0352a19f80 x1855007617379072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077358.163880:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617379072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077358.163884:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077358.163886:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077358.163889:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077358.675356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077359.059348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077359.059401:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077359.059474:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077359.699362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077360.083324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077360.083834:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077360.083910:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077360.723358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077361.107545:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077361.107704:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077361.108312:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077361.747317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077362.131474:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077362.131509:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077362.131691:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077362.771333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077363.155339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077363.155341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077363.155390:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077363.283342:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077363.283351:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077363.283355:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077363.283364:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4352 00000100:00080000:3.0:1769077363.283373:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077363.283379:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077363.283419:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617379200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077363.283451:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077363.283454:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077363.283456:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077363.283459:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4352 00000100:00080000:3.0:1769077363.283463:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077363.283466:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077363.283474:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077363.283484:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617379328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077363.283491:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077363.283494:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077363.283495:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077363.283498:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4352 00000100:00080000:3.0:1769077363.283500:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077363.283503:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077363.283506:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077363.283514:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617379200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077363.283515:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617379456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077363.283528:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077363.283530:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077363.283531:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077363.283535:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4352 00000100:00080000:3.0:1769077363.283537:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077363.283549:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077363.283558:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077363.283571:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617379584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077363.283581:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077363.283587:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1769077363.283588:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077363.283592:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077363.283599:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4352 00000100:00080000:3.0:1769077363.283603:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077363.283607:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077363.283611:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077363.283636:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617379712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077363.283667:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077363/real 1769077363] req@ffff9b0352a19f80 x1855007617379200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077363.283689:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617379200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077363.283700:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077363.283705:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077363.283717:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617379200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077363.283727:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077363.283747:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077363.283756:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077363/real 1769077363] req@ffff9b0352a1a300 x1855007617379328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077363.283764:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617379328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077363.283778:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077363.283781:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077363.283786:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077363.283804:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077363.283816:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077363.283823:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077363/real 1769077363] req@ffff9b0352a18e00 x1855007617379456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077363.283830:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617379456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077363.283835:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077363.283837:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077363.283839:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077363.283843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077363.283852:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077363.283857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077363/real 1769077363] req@ffff9b0352a1aa00 x1855007617379584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077363.283864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617379584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077363.283869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077363.283870:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077363.283872:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077363.283875:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077363.283884:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077363.283889:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077363/real 1769077363] req@ffff9b0352a18700 x1855007617379712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077363.283896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617379712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077363.283901:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077363.283903:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077363.283905:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077363.795342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077364.179876:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077364.179969:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077364.180055:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077364.819355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077365.203325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077365.203393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077365.203417:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077365.843365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077366.227367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077366.227381:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077366.227449:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077366.867345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077367.251337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077367.251342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077367.251365:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077367.891349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077368.275358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077368.275358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077368.275378:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077368.403334:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077368.403343:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077368.403347:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077368.403356:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4357 00000100:00080000:3.0:1769077368.403365:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077368.403371:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077368.403415:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617379840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077368.403437:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077368.403439:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077368.403441:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077368.403444:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4357 00000100:00080000:3.0:1769077368.403446:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077368.403449:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077368.403456:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077368.403466:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617379968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077368.403474:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077368.403476:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077368.403478:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077368.403480:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4357 00000100:00080000:3.0:1769077368.403482:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077368.403487:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077368.403489:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617379840:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769077368.403491:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077368.403502:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617380096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077368.403509:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077368.403511:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077368.403512:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077368.403515:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4357 00000100:00080000:3.0:1769077368.403517:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077368.403519:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077368.403522:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077368.403554:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617380224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077368.403561:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077368.403563:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077368.403564:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077368.403566:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4357 00000100:00080000:3.0:1769077368.403570:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077368.403574:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077368.403577:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077368.403582:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077368.403592:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617380352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077368.403615:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077368/real 1769077368] req@ffff9b0352a1b100 x1855007617379840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077368.403631:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617379840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077368.403639:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077368.403642:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077368.403645:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617379840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077368.403650:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077368.403666:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077368.403673:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077368/real 1769077368] req@ffff9b0352a1a680 x1855007617379968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077368.403682:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617379968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077368.403687:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077368.403689:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077368.403692:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617379968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077368.403710:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077368.403720:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077368.403726:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077368/real 1769077368] req@ffff9b0352a18a80 x1855007617380096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077368.403734:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617380096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077368.403739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077368.403740:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077368.403742:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077368.403746:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077368.403756:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077368.403761:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077368/real 1769077368] req@ffff9b0352a1b480 x1855007617380224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077368.403768:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617380224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077368.403778:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077368.403780:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077368.403782:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077368.403786:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077368.403794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077368.403799:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077368/real 1769077368] req@ffff9b0352a1ad80 x1855007617380352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077368.403806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617380352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077368.403810:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077368.403812:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077368.403814:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077368.915317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077369.299336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077369.299336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077369.299427:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077369.939342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077370.323318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077370.323318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077370.323375:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077370.963340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077371.347338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077371.347338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077371.347360:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077371.987337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077372.371324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077372.371324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077372.371351:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077373.011400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077373.395363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077373.395393:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077373.395430:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:0.0:1769077373.523360:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077373.523372:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077373.523377:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077373.523388:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4362 00000100:00080000:0.0:1769077373.523399:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077373.523406:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077373.523481:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617380480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077373.523508:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077373.523512:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077373.523513:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077373.523516:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4362 00000100:00080000:0.0:1769077373.523520:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077373.523523:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077373.523531:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077373.523543:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617380608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077373.523552:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077373.523555:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077373.523557:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077373.523560:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4362 00000100:00080000:0.0:1769077373.523562:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077373.523565:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077373.523568:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077373.523574:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617380480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077373.523579:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617380736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077373.523588:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077373.523590:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077373.523591:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077373.523594:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4362 00000100:00080000:0.0:1769077373.523596:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077373.523599:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077373.523602:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077373.523611:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617380864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077373.523616:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077373.523619:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077373.523620:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077373.523623:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4362 00000100:00080000:0.0:1769077373.523626:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077373.523628:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077373.523631:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077373.523639:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617380992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077373.523667:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077373.523731:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077373/real 1769077373] req@ffff9b0365a38a80 x1855007617380480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077373.523755:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617380480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077373.523768:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077373.523773:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077373.523778:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617380480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077373.523807:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077373.523835:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077373.523850:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077373/real 1769077373] req@ffff9b0365a39c00 x1855007617380608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077373.523864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617380608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077373.523874:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077373.523878:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077373.523881:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077373.523887:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077373.523903:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077373.523911:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077373/real 1769077373] req@ffff9b0365a38e00 x1855007617380736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077373.523919:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617380736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077373.523925:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077373.523928:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077373.523930:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077373.523934:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077373.523946:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077373.523952:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077373/real 1769077373] req@ffff9b0365a3b480 x1855007617380864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077373.523959:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617380864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077373.523965:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077373.523967:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077373.523969:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077373.523972:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077373.523981:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077373.523986:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077373/real 1769077373] req@ffff9b0365a3aa00 x1855007617380992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077373.523994:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617380992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077373.523999:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077373.524000:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077373.524002:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617380992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077374.035341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077374.419343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077374.419343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077374.419443:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077375.059308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077375.443314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077375.443321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077375.443401:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077376.083356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077376.467321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077376.467346:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077376.467349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077377.107371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077377.491347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077377.491347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077377.491416:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077378.131404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077378.515305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077378.515319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077378.515354:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077378.643367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077378.643376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077378.643384:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077378.643393:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4367 00000100:00080000:0.0:1769077378.643402:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077378.643408:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077378.643441:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617381120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077378.643462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077378.643464:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077378.643466:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077378.643469:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4367 00000100:00080000:0.0:1769077378.643472:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077378.643475:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077378.643483:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077378.643491:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617381120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077378.643493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617381248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077378.643511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077378.643513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077378.643515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077378.643517:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4367 00000100:00080000:0.0:1769077378.643520:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077378.643522:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077378.643525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077378.643533:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617381376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077378.643539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077378.643542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077378.643543:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077378.643546:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4367 00000100:00080000:0.0:1769077378.643556:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077378.643558:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077378.643562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077378.643569:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617381504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077378.643576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077378.643578:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077378.643580:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077378.643582:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4367 00000100:00080000:0.0:1769077378.643586:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077378.643590:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077378.643593:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077378.643605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617381632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077378.643605:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617381248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077378.643639:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617381376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077378.643659:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617381504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077378.643684:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617381632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077379.155316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077379.539381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077379.539381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077379.539457:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077380.179351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077380.563301:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077380.563359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077380.563433:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077381.203329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077381.587361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077381.587361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077381.587389:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769077381.715516:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:3.0:1769077381.715552:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077381.715567:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769077381.715607:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769077381.715617:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769077381.715627:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769077381.715638:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769077381.715656:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077378/real 1769077381] req@ffff9b0365a39180 x1855007617381248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077381.715684:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617381248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077381.715696:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077381.715702:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077381.715707:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617381248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077381.715714:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077378/real 1769077381] req@ffff9b0365a3ad80 x1855007617381376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077381.715726:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617381376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077381.715734:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077381.715738:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077381.715742:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617381376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077381.715752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077378/real 1769077381] req@ffff9b0365a3bb80 x1855007617381504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077381.715767:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617381504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077381.715775:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077381.715778:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077381.715781:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617381504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077381.715787:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077378/real 1769077381] req@ffff9b0365a39500 x1855007617381120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077381.715798:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617381120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077381.715806:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077381.715809:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077381.715813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617381120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077381.715839:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077378/real 1769077381] req@ffff9b0365a3b100 x1855007617381632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077381.715852:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617381632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077381.715861:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077381.715865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077381.715869:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617381632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077382.227324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077382.611340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077382.611353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077382.611409:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077383.251326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077383.635344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077383.635352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077383.635400:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077383.763356:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077383.763366:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077383.763371:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077383.763382:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4372 00000100:00080000:1.0:1769077383.763392:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077383.763410:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077383.763446:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617381760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077383.763475:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077383.763479:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077383.763482:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077383.763487:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4372 00000100:00080000:1.0:1769077383.763492:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077383.763499:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077383.763510:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077383.763519:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617381760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077383.763526:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617381888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077383.763543:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077383.763547:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077383.763549:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077383.763554:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4372 00000100:00080000:1.0:1769077383.763559:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077383.763563:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077383.763569:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077383.763582:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617382016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077383.763594:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077383.763595:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077383.763602:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077383.763605:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077383.763610:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4372 00000100:00080000:1.0:1769077383.763617:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077383.763622:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077383.763628:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077383.763645:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077383/real 1769077383] req@ffff9b0343dedf80 x1855007617381760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077383.763646:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617382144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077383.763681:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077383.763686:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077383.763689:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077383.763691:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617381760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077383.763696:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4372 00000100:00080000:1.0:1769077383.763704:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077383.763710:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077383.763711:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:0.0:1769077383.763722:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000008:00100000:1.0:1769077383.763722:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077383.763731:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617381760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077383.763746:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617382272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077383.763761:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617381888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077383.763789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077383.763803:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077383/real 1769077383] req@ffff9b0343def100 x1855007617381888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077383.763816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617381888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077383.763826:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077383.763830:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077383.763834:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617381888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077383.763843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077383.763864:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077383.763875:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077383/real 1769077383] req@ffff9b0343dee300 x1855007617382016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077383.763888:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617382016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077383.763896:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077383.763900:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077383.763904:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077383.763910:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077383.763929:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077383.763938:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077383/real 1769077383] req@ffff9b0343dec380 x1855007617382144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077383.763950:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617382144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077383.763960:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077383.763963:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077383.763967:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077383.763983:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077383.764002:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077383.764012:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077383/real 1769077383] req@ffff9b0343ded180 x1855007617382272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077383.764044:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617382272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077383.764052:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077383.764056:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077383.764059:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077384.275319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077384.659360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077384.659365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077384.659432:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077385.299374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077385.683359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077385.683378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077385.683425:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077386.323307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077386.707335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077386.707357:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077386.707511:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077387.347305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077387.731316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077387.731317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077387.731372:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077388.371338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077388.755357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077388.755371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077388.755426:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077388.883387:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077388.883403:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077388.883409:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077388.883421:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4378 00000100:00080000:1.0:1769077388.883434:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077388.883444:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077388.883492:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617382400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077388.883525:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077388.883529:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077388.883530:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077388.883541:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4378 00000100:00080000:1.0:1769077388.883548:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077388.883553:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077388.883567:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077388.883580:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617382528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077388.883590:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077388.883594:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077388.883596:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077388.883601:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4378 00000100:00080000:1.0:1769077388.883604:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077388.883611:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077388.883613:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617382400:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769077388.883617:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077388.883635:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617382656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077388.883645:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077388.883648:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077388.883650:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077388.883654:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4378 00000100:00080000:1.0:1769077388.883656:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077388.883659:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077388.883662:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077388.883668:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617382784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077388.883675:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077388.883677:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077388.883679:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077388.883700:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4378 00000100:00080000:1.0:1769077388.883703:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077388.883706:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077388.883712:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077388.883728:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617382912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077388.883882:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077388.883953:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077388/real 1769077388] req@ffff9b0343dec700 x1855007617382400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077388.883971:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617382400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077388.883979:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077388.883983:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077388.883987:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617382400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077388.883996:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077388.884021:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077388.884036:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077388/real 1769077388] req@ffff9b0343dec000 x1855007617382528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077388.884050:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617382528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077388.884058:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077388.884060:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077388.884063:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077388.884084:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077388.884101:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077388.884108:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077388/real 1769077388] req@ffff9b0343ded180 x1855007617382656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077388.884116:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617382656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077388.884121:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077388.884123:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077388.884125:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077388.884129:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077388.884139:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077388.884145:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077388/real 1769077388] req@ffff9b0343dec380 x1855007617382784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077388.884152:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617382784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077388.884158:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077388.884160:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077388.884162:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077388.884168:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077388.884204:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077388.884217:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077388/real 1769077388] req@ffff9b0343dee300 x1855007617382912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077388.884230:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617382912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077388.884271:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077388.884276:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077388.884283:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617382912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077389.395351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077389.779321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077389.779349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077389.779404:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077390.419331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077390.803349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077390.803349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077390.803401:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077391.443344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077391.827364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077391.827371:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077391.827421:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077392.467286:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077392.851340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077392.851340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077392.851358:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077393.491340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077393.875336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077393.875336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077393.875423:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077394.003340:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077394.003352:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077394.003356:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077394.003368:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4383 00000100:00080000:1.0:1769077394.003378:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077394.003385:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077394.003423:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617383040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077394.003447:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077394.003450:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077394.003451:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077394.003454:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4383 00000100:00080000:1.0:1769077394.003457:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077394.003461:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077394.003469:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077394.003478:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617383168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077394.003487:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077394.003491:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077394.003494:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077394.003497:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617383040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077394.003500:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4383 00000100:00080000:1.0:1769077394.003508:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077394.003514:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077394.003520:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077394.003534:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617383296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077394.003545:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077394.003548:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077394.003549:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077394.003552:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4383 00000100:00080000:1.0:1769077394.003563:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077394.003566:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077394.003570:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077394.003572:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077394.003583:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617383424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077394.003592:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077394.003594:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077394.003595:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077394.003598:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4383 00000100:00080000:1.0:1769077394.003600:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077394.003603:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077394.003606:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077394.003612:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9b0343def100 x1855007617383040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077394.003616:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617383552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077394.003652:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617383040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077394.003662:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077394.003666:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077394.003669:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617383040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077394.003687:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077394.003703:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077394.003711:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9b0343dedf80 x1855007617383168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077394.003719:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617383168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077394.003725:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077394.003727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077394.003729:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077394.003733:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077394.003742:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077394.003747:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9b0343ded880 x1855007617383296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077394.003755:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617383296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077394.003760:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077394.003762:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077394.003764:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077394.003768:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077394.003778:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077394.003784:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9b0343def800 x1855007617383424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077394.003791:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617383424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077394.003796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077394.003798:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077394.003800:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077394.003803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077394.003812:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077394.003817:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077394/real 1769077394] req@ffff9b0343deea00 x1855007617383552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077394.003828:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617383552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077394.003836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077394.003839:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077394.003843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077394.515365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077394.899357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077394.899381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077394.899440:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077395.539393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077395.923354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077395.923399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077395.923411:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077396.563396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077396.947349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077396.947349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077396.947412:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077397.587387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077397.971318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077397.971318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077397.971348:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077398.611353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077398.995358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077398.995358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077398.995431:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077399.123323:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077399.123331:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077399.123335:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077399.123343:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4388 00000100:00080000:1.0:1769077399.123351:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077399.123357:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077399.123388:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617383680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077399.123413:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077399.123417:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077399.123419:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077399.123424:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4388 00000100:00080000:1.0:1769077399.123428:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077399.123432:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077399.123442:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077399.123455:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617383808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077399.123464:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617383680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077399.123465:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077399.123468:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077399.123470:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077399.123473:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4388 00000100:00080000:1.0:1769077399.123477:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077399.123480:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077399.123485:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077399.123493:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617383936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077399.123501:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077399.123504:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077399.123505:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077399.123508:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4388 00000100:00080000:1.0:1769077399.123511:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077399.123514:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077399.123518:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077399.123525:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077399.123526:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617384064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077399.123539:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077399.123542:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077399.123544:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077399.123547:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4388 00000100:00080000:1.0:1769077399.123550:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077399.123554:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077399.123557:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077399.123561:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077399/real 1769077399] req@ffff9b0343deca80 x1855007617383680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077399.123566:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617384192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077399.123580:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617383680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077399.123590:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077399.123595:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077399.123599:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617383680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077399.123633:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077399.123652:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077399.123661:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077399/real 1769077399] req@ffff9b0343deed80 x1855007617383808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077399.123670:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617383808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077399.123676:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077399.123678:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077399.123681:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077399.123684:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077399.123695:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077399.123701:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077399/real 1769077399] req@ffff9b0343dee680 x1855007617383936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077399.123709:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617383936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077399.123713:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077399.123715:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077399.123717:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617383936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077399.123721:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077399.123730:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077399.123736:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077399/real 1769077399] req@ffff9b0343defb80 x1855007617384064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077399.123744:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617384064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077399.123749:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077399.123750:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077399.123752:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077399.123755:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077399.123764:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077399.123769:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077399/real 1769077399] req@ffff9b0343dece00 x1855007617384192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077399.123776:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617384192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077399.123781:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077399.123783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077399.123785:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077399.635348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077400.019312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077400.019312:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077400.019400:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077400.659369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077401.043332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077401.043332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077401.043444:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077401.683330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077402.067313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077402.067352:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077402.067369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077402.707335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077403.091329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077403.091331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077403.091363:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077403.731333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077404.115337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077404.115337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077404.115400:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077404.243331:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077404.243340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077404.243344:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077404.243353:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4393 00000100:00080000:0.0:1769077404.243362:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077404.243369:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077404.243401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617384320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077404.243425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077404.243428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077404.243433:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077404.243436:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4393 00000100:00080000:0.0:1769077404.243439:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077404.243442:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077404.243449:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077404.243459:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617384448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077404.243469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077404.243473:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077404.243476:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077404.243484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4393 00000100:00100000:3.0:1769077404.243485:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617384320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077404.243488:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077404.243496:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077404.243503:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077404.243518:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617384576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077404.243529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077404.243533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077404.243535:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077404.243538:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4393 00000100:00080000:0.0:1769077404.243542:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077404.243545:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077404.243549:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769077404.243560:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077404.243576:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617384704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077404.243584:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077404.243587:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077404/real 1769077404] req@ffff9b0365a3b100 x1855007617384320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077404.243588:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077404.243592:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077404.243600:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4393 00000100:00080000:0.0:1769077404.243604:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077404.243607:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077404.243612:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077404.243622:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617384320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077404.243632:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077404.243636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617384832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077404.243637:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077404.243643:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617384320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077404.243650:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077404.243679:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077404.243695:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077404/real 1769077404] req@ffff9b0365a39500 x1855007617384448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077404.243708:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617384448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077404.243718:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077404.243722:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077404.243725:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077404.243751:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077404.243769:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077404.243777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077404/real 1769077404] req@ffff9b0365a3bb80 x1855007617384576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077404.243785:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617384576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077404.243791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077404.243793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077404.243795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077404.243799:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077404.243809:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077404.243814:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077404/real 1769077404] req@ffff9b0365a3ad80 x1855007617384704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077404.243822:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617384704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077404.243827:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077404.243828:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077404.243831:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077404.243835:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077404.243844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077404.243849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077404/real 1769077404] req@ffff9b0365a39180 x1855007617384832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077404.243856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617384832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077404.243861:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077404.243863:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077404.243864:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617384832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077404.755307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077405.139325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077405.139325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077405.139391:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077405.779343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077406.163348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077406.163348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077406.163410:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077406.803446:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077407.187360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077407.187360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077407.187388:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077407.827326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077408.211376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077408.211388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077408.211415:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077408.851387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077409.235337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077409.235383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077409.235415:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077409.363345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077409.363355:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077409.363360:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077409.363370:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4398 00000100:00080000:0.0:1769077409.363379:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077409.363385:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077409.363437:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617384960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077409.363462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077409.363465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077409.363467:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077409.363470:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4398 00000100:00080000:0.0:1769077409.363473:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077409.363475:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077409.363482:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077409.363492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617385088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077409.363507:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077409.363510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077409.363511:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077409.363518:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4398 00000100:00100000:3.0:1769077409.363518:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617384960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077409.363523:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077409.363528:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077409.363532:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077409.363541:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617385216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077409.363547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077409.363549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077409.363551:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077409.363554:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4398 00000100:00080000:0.0:1769077409.363557:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077409.363559:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077409.363563:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077409.363570:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617385344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077409.363577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077409.363579:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077409.363581:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077409.363584:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4398 00000100:00080000:0.0:1769077409.363587:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077409.363590:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077409.363594:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077409.363597:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077409.363605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617385472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077409.363641:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077409/real 1769077409] req@ffff9b0365a39180 x1855007617384960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077409.363658:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617384960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077409.363666:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077409.363670:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077409.363674:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617384960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077409.363693:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077409.363712:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077409.363721:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077409/real 1769077409] req@ffff9b0365a3ad80 x1855007617385088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077409.363730:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617385088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077409.363736:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077409.363739:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077409.363741:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077409.363745:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077409.363756:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077409.363763:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077409/real 1769077409] req@ffff9b0365a3bb80 x1855007617385216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077409.363771:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617385216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077409.363777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077409.363780:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077409.363784:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077409.363791:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077409.363814:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077409.363828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077409/real 1769077409] req@ffff9b0365a39500 x1855007617385344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077409.363871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617385344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077409.363879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077409.363882:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077409.363884:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077409.363889:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077409.363905:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077409.363913:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077409/real 1769077409] req@ffff9b0365a3b100 x1855007617385472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077409.363922:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617385472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077409.363927:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077409.363930:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077409.363932:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077409.875339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077410.259336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077410.259336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077410.259432:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077410.899346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077411.283336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077411.283405:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077411.283495:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077411.923316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077412.307344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077412.307347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077412.307401:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077412.947340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077413.331325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077413.331360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077413.331369:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077413.971356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077414.355344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077414.355344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077414.355429:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077414.483354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077414.483363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077414.483367:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077414.483376:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4403 00000100:00080000:0.0:1769077414.483384:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077414.483392:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077414.483430:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617385600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077414.483462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077414.483465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077414.483466:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077414.483470:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4403 00000100:00080000:0.0:1769077414.483473:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077414.483477:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077414.483484:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077414.483499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617385728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077414.483506:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077414.483509:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077414.483510:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077414.483512:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4403 00000100:00080000:0.0:1769077414.483515:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077414.483518:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077414.483522:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077414.483533:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617385856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077414.483543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077414.483546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077414.483548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077414.483552:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4403 00000100:00080000:0.0:1769077414.483559:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077414.483561:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617385600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077414.483567:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077414.483578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077414.483591:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617385984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077414.483599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077414.483602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077414.483603:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077414.483606:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4403 00000100:00080000:0.0:1769077414.483609:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077414.483612:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077414.483615:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077414.483624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617386112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077414.483630:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077414.483671:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077414/real 1769077414] req@ffff9b0365a3aa00 x1855007617385600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077414.483686:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617385600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077414.483694:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077414.483697:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077414.483700:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617385600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077414.483705:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077414.483719:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077414.483726:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077414/real 1769077414] req@ffff9b0365a3b480 x1855007617385728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077414.483734:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617385728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077414.483739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077414.483741:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077414.483743:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077414.483781:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077414.483796:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077414.483803:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077414/real 1769077414] req@ffff9b0365a38e00 x1855007617385856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077414.483816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617385856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077414.483821:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077414.483823:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077414.483825:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077414.483829:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077414.483838:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077414.483844:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077414/real 1769077414] req@ffff9b0365a39c00 x1855007617385984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077414.483851:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617385984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077414.483856:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077414.483857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077414.483859:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617385984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077414.483862:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617386112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077414.483871:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077414.483877:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077414/real 1769077414] req@ffff9b0365a38a80 x1855007617386112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077414.483884:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617386112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077414.483893:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077414.483895:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077414.483897:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617386112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077414.995346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077415.379317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077415.379320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077415.379387:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077416.019342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077416.403406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077416.403413:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077416.403477:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077417.043331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077417.427325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077417.427336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077417.427351:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077418.067345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077418.451343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077418.451343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077418.451411:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077419.091337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077419.475302:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077419.475389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077419.475416:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077419.603387:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077419.603396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077419.603400:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077419.603409:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4408 00000100:00080000:3.0:1769077419.603417:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077419.603422:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077419.603461:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617386240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077419.603491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077419.603495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077419.603497:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077419.603501:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4408 00000100:00080000:3.0:1769077419.603507:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077419.603511:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077419.603521:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077419.603531:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617386240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077419.603536:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617386368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077419.603549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077419.603554:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077419.603556:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077419.603560:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4408 00000100:00080000:3.0:1769077419.603565:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077419.603569:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077419.603575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077419.603586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617386496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077419.603595:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077419.603602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077419.603606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077419.603608:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077419.603614:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4408 00000100:00080000:3.0:1769077419.603618:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077419.603622:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077419.603628:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077419.603640:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077419/real 1769077419] req@ffff9b0352a18000 x1855007617386240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077419.603642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617386624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077419.603684:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077419.603687:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077419.603688:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077419.603696:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4408 00000100:00080000:3.0:1769077419.603700:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077419.603704:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077419.603709:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077419.603712:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617386240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077419.603721:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617386752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077419.603724:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077419.603734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077419.603739:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617386240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077419.603759:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617386368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077419.603777:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077419.603787:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077419/real 1769077419] req@ffff9b0352a18380 x1855007617386368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077419.603795:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617386368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077419.603801:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077419.603804:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077419.603806:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617386368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077419.603810:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617386496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077419.603821:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077419.603827:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077419/real 1769077419] req@ffff9b0352a19c00 x1855007617386496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077419.603834:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617386496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077419.603839:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077419.603841:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077419.603843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617386496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077419.603847:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617386624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077419.603858:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077419.603863:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077419/real 1769077419] req@ffff9b0352a1b800 x1855007617386624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077419.603871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617386624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077419.603876:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077419.603878:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077419.603880:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617386624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077419.603883:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617386752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077419.603892:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077419.603900:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077419/real 1769077419] req@ffff9b0352a19500 x1855007617386752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077419.603913:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617386752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077419.603921:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077419.603924:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077419.603928:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617386752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077420.115345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077420.499336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077420.499347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077420.499425:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077421.139326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077421.523305:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077421.523338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077421.523400:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077422.163338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077422.547353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077422.547363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077422.547384:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077423.187347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077423.571350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077423.571396:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077423.571401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077424.211359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077424.595361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077424.595361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077424.595415:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077424.723330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077424.723340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077424.723344:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077424.723353:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4413 00000100:00080000:3.0:1769077424.723362:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077424.723368:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077424.723399:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617386880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077424.723428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077424.723432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077424.723434:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077424.723439:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4413 00000100:00080000:3.0:1769077424.723443:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077424.723448:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077424.723458:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077424.723474:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617387008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077424.723485:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077424.723487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077424.723489:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077424.723492:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4413 00000100:00080000:3.0:1769077424.723495:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077424.723497:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077424.723501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077424.723510:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617387136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077424.723517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077424.723520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077424.723521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077424.723524:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4413 00000100:00080000:3.0:1769077424.723526:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077424.723528:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077424.723533:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077424.723534:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617386880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077424.723542:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617387264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077424.723551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077424.723553:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077424.723555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077424.723558:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4413 00000100:00080000:3.0:1769077424.723560:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077424.723562:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077424.723565:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077424.723575:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617387392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077424.723616:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077424.723655:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077424/real 1769077424] req@ffff9b0352a19500 x1855007617386880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077424.723672:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617386880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077424.723680:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077424.723684:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077424.723687:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617386880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077424.723693:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077424.723733:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077424.723744:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077424/real 1769077424] req@ffff9b0352a1b800 x1855007617387008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077424.723754:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617387008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077424.723761:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077424.723764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077424.723767:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077424.723790:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077424.723805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077424.723813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077424/real 1769077424] req@ffff9b0352a19c00 x1855007617387136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077424.723829:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617387136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077424.723834:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077424.723836:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077424.723838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077424.723859:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077424.723871:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077424.723877:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077424/real 1769077424] req@ffff9b0352a18380 x1855007617387264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077424.723885:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617387264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077424.723890:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077424.723891:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077424.723893:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077424.723896:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077424.723905:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077424.723910:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077424/real 1769077424] req@ffff9b0352a18000 x1855007617387392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077424.723917:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617387392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077424.723927:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077424.723929:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077424.723931:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077425.235341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077425.619452:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077425.620414:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077425.620437:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077426.259344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077426.643351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077426.643379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077426.643459:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077427.283353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077427.667388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077427.667406:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077427.667554:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077428.307343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077428.691364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077428.691618:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077428.691848:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077429.331349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077429.715376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077429.715422:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077429.715433:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769077429.843347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077429.843356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077429.843360:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077429.843369:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4419 00000100:00080000:3.0:1769077429.843376:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077429.843382:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077429.843414:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617387520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077429.843438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077429.843441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077429.843442:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077429.843445:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4419 00000100:00080000:3.0:1769077429.843447:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077429.843450:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077429.843464:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077429.843475:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617387648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077429.843487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077429.843489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077429.843492:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077429.843508:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617387520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077429.843509:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4419 00000100:00080000:3.0:1769077429.843514:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077429.843519:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077429.843524:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077429.843534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617387776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077429.843541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077429.843544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077429.843545:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077429.843548:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4419 00000100:00080000:3.0:1769077429.843551:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077429.843553:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077429.843556:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077429.843565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617387904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077429.843571:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077429.843575:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077429.843578:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077429.843579:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077429.843584:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4419 00000100:00080000:3.0:1769077429.843587:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077429.843590:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077429.843594:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077429.843602:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617388032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077429.843611:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077429/real 1769077429] req@ffff9b0352a1ad80 x1855007617387520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077429.843626:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617387520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077429.843635:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077429.843638:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077429.843641:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617387520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077429.843659:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077429.843674:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077429.843681:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077429/real 1769077429] req@ffff9b0352a1b480 x1855007617387648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077429.843689:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617387648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077429.843705:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077429.843707:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077429.843710:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077429.843713:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077429.843722:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077429.843727:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077429/real 1769077429] req@ffff9b0352a18a80 x1855007617387776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077429.843734:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617387776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077429.843739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077429.843741:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077429.843743:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077429.843746:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077429.843756:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077429.843761:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077429/real 1769077429] req@ffff9b0352a1a680 x1855007617387904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077429.843783:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617387904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077429.843788:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077429.843790:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077429.843791:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617387904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077429.843795:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077429.843803:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077429.843808:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077429/real 1769077429] req@ffff9b0352a1b100 x1855007617388032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077429.843815:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617388032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077429.843819:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077429.843821:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077429.843823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077430.355417:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077430.739494:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077430.739518:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077430.739555:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077431.379309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077431.763306:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077431.763315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077431.763364:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077432.403362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077432.787348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077432.787348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077432.787404:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077433.427300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077433.811392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077433.811400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077433.811469:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077434.451413:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077434.835335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077434.835335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077434.835407:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077434.963347:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077434.963356:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077434.963361:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077434.963369:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4424 00000100:00080000:1.0:1769077434.963378:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077434.963384:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077434.963427:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617388160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077434.963448:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077434.963451:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077434.963452:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077434.963456:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4424 00000100:00080000:1.0:1769077434.963458:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077434.963462:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077434.963469:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077434.963479:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617388288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077434.963486:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077434.963489:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077434.963490:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077434.963493:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4424 00000100:00080000:1.0:1769077434.963495:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077434.963501:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617388160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077434.963503:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077434.963511:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077434.963520:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617388416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077434.963528:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077434.963530:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077434.963531:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077434.963534:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4424 00000100:00080000:1.0:1769077434.963536:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077434.963539:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077434.963545:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077434.963552:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617388544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077434.963558:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077434.963561:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077434.963562:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077434.963565:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4424 00000100:00080000:1.0:1769077434.963575:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077434.963577:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077434.963580:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769077434.963586:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077434.963590:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617388672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077434.963625:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077434/real 1769077434] req@ffff9b0343ded500 x1855007617388160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077434.963639:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617388160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077434.963652:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077434.963655:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077434.963658:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617388160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077434.963664:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077434.963680:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077434.963687:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077434/real 1769077434] req@ffff9b0343def480 x1855007617388288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077434.963695:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617388288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077434.963701:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077434.963703:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077434.963705:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077434.963723:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077434.963734:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077434.963739:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077434/real 1769077434] req@ffff9b0343dece00 x1855007617388416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077434.963747:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617388416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077434.963752:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077434.963754:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077434.963755:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077434.963760:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077434.963769:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077434.963775:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077434/real 1769077434] req@ffff9b0343defb80 x1855007617388544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077434.963782:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617388544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077434.963787:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077434.963788:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077434.963790:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077434.963794:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077434.963802:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077434.963811:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077434/real 1769077434] req@ffff9b0343dee680 x1855007617388672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077434.963823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617388672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077434.963831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077434.963834:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077434.963837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077435.475339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077435.859353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077435.859353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077435.859439:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077436.499349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077436.883335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077436.883335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077436.883399:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077437.523349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077437.907349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077437.907414:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077437.907441:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077438.547380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077438.931361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077438.931361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077438.931460:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077439.571389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077439.955357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077439.955357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077439.955447:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077440.083352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077440.083362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077440.083366:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077440.083375:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4429 00000100:00080000:3.0:1769077440.083384:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077440.083389:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077440.083422:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617388800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077440.083449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077440.083453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077440.083456:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077440.083460:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4429 00000100:00080000:3.0:1769077440.083464:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077440.083468:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077440.083478:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077440.083496:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617388928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077440.083498:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617388800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077440.083514:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077440.083517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077440.083518:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077440.083522:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4429 00000100:00080000:3.0:1769077440.083526:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077440.083530:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077440.083534:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077440.083544:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617389056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077440.083551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077440.083553:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077440.083555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077440.083560:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077440.083560:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4429 00000100:00080000:3.0:1769077440.083566:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077440.083569:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077440.083573:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077440.083583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617389184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077440.083590:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077440.083592:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077440.083594:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077440.083597:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4429 00000100:00080000:3.0:1769077440.083600:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077440.083602:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077440.083605:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077440.083609:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077440/real 1769077440] req@ffff9b0352a18700 x1855007617388800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077440.083617:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617389312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077440.083646:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617388800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077440.083659:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077440.083665:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077440.083669:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617388800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077440.083691:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077440.083711:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077440.083721:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077440/real 1769077440] req@ffff9b0352a1aa00 x1855007617388928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077440.083729:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617388928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077440.083735:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077440.083738:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077440.083740:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617388928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077440.083744:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077440.083754:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077440.083760:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077440/real 1769077440] req@ffff9b0352a18e00 x1855007617389056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077440.083767:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617389056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077440.083777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077440.083779:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077440.083781:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077440.083784:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077440.083793:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077440.083798:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077440/real 1769077440] req@ffff9b0352a1a300 x1855007617389184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077440.083805:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617389184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077440.083810:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077440.083812:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077440.083814:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077440.083817:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077440.083825:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077440.083830:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077440/real 1769077440] req@ffff9b0352a19f80 x1855007617389312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077440.083837:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617389312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077440.083841:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077440.083843:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077440.083846:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077440.595372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077440.979329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077440.979370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077440.979383:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077441.619499:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077442.003359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077442.003364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077442.003425:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077442.643377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077443.027339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077443.027341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077443.027393:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077443.667403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077444.051317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077444.051321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077444.051385:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077444.691360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077445.075334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077445.075425:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077445.075457:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769077445.203363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077445.203376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077445.203381:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077445.203393:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4434 00000100:00080000:3.0:1769077445.203404:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077445.203412:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077445.203457:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617389440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077445.203484:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077445.203487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077445.203490:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077445.203494:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4434 00000100:00080000:3.0:1769077445.203496:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077445.203499:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077445.203507:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077445.203520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617389568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077445.203527:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077445.203531:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077445.203532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077445.203535:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4434 00000100:00080000:3.0:1769077445.203539:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077445.203543:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077445.203546:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077445.203554:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617389696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077445.203564:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617389440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077445.203566:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077445.203572:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077445.203583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077445.203588:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4434 00000100:00080000:3.0:1769077445.203590:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077445.203593:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077445.203601:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077445.203615:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617389824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077445.203621:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077445.203624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077445.203626:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077445.203628:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4434 00000100:00080000:3.0:1769077445.203633:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077445.203636:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077445.203639:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077445.203646:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617389952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077445.203695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077445.203731:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077445.203747:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077445.203762:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077445.715312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077446.099366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077446.099373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077446.099442:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077446.739350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077447.123423:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077447.123423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077447.123499:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077447.763326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077448.147363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077448.147369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077448.147447:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769077448.275601:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769077448.275657:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769077448.275676:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077448.275719:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077448.275728:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077448.275736:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077448.275744:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:3.0:1769077448.275972:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077445/real 1769077448] req@ffff9b0352a18e00 x1855007617389696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077448.275997:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617389696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077448.276009:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077448.276015:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077448.276020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077448.276045:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077445/real 1769077448] req@ffff9b0352a1aa00 x1855007617389824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077448.276065:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617389824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077448.276073:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077448.276077:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077448.276081:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077448.276088:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077445/real 1769077448] req@ffff9b0352a18700 x1855007617389952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077448.276100:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617389952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077448.276109:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077448.276112:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077448.276116:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077448.276122:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077445/real 1769077448] req@ffff9b0352a19f80 x1855007617389440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077448.276135:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617389440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077448.276144:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077448.276148:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077448.276151:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617389440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077448.276159:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077445/real 1769077448] req@ffff9b0352a1a300 x1855007617389568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077448.276180:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617389568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077448.276203:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077448.276206:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077448.276209:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617389568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077448.787324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077449.171366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077449.171366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077449.171407:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077449.811361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077450.195332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077450.195349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077450.195420:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077450.323336:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077450.323345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077450.323349:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077450.323357:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4439 00000100:00080000:3.0:1769077450.323367:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077450.323374:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077450.323407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617390080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077450.323440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077450.323444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077450.323446:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077450.323451:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4439 00000100:00080000:3.0:1769077450.323455:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077450.323460:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077450.323470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077450.323486:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617390208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077450.323493:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617390080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077450.323515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077450.323520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077450.323522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077450.323528:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4439 00000100:00080000:3.0:1769077450.323532:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077450.323537:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077450.323542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077450.323555:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617390336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077450.323567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077450.323574:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077450.323575:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077450.323579:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077450.323584:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4439 00000100:00080000:3.0:1769077450.323589:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077450.323594:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077450.323599:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077450.323612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617390464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077450.323623:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077450/real 1769077450] req@ffff9b0352a1a300 x1855007617390080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077450.323624:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077450.323632:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077450.323636:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077450.323645:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4439 00000100:00080000:3.0:1769077450.323651:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077450.323659:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077450.323660:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617390080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077450.323667:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1769077450.323680:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077450.323686:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617390592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077450.323690:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077450.323699:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617390080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077450.323737:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077450.323763:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077450.323778:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077450/real 1769077450] req@ffff9b0352a19f80 x1855007617390208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077450.323790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617390208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077450.323798:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077450.323802:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077450.323806:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077450.323813:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077450.323833:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077450.323845:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077450/real 1769077450] req@ffff9b0352a18700 x1855007617390336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077450.323856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617390336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077450.323865:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077450.323868:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077450.323871:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077450.323877:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077450.323898:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077450.323910:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077450/real 1769077450] req@ffff9b0352a1aa00 x1855007617390464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077450.323923:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617390464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077450.323931:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077450.323935:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077450.323938:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077450.323945:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077450.323964:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077450.323975:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077450/real 1769077450] req@ffff9b0352a18e00 x1855007617390592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077450.323987:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617390592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077450.323995:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077450.323998:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077450.324001:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077450.835352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077451.219396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077451.219410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077451.219479:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077451.859421:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077452.243381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077452.243381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077452.243442:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077452.883319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077453.267367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077453.267367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077453.267450:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077453.907305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077454.291320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077454.291321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077454.291344:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077454.931300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077455.315345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077455.315345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077455.315401:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077455.443338:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077455.443348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077455.443353:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077455.443364:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4444 00000100:00080000:3.0:1769077455.443373:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077455.443381:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077455.443417:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617390720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077455.443448:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077455.443452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077455.443454:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077455.443460:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4444 00000100:00080000:3.0:1769077455.443464:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077455.443469:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077455.443478:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077455.443492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617390848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077455.443493:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617390720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077455.443511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077455.443515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077455.443517:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077455.443521:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4444 00000100:00080000:3.0:1769077455.443525:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077455.443529:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077455.443533:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077455.443545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617390976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077455.443554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077455.443558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077455.443560:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077455.443564:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077455.443569:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4444 00000100:00080000:3.0:1769077455.443574:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077455.443579:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077455.443585:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077455.443597:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617391104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077455.443607:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077455.443612:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077455.443613:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077455/real 1769077455] req@ffff9b0352a1b100 x1855007617390720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077455.443615:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077455.443624:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4444 00000100:00080000:3.0:1769077455.443630:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077455.443638:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077455.443644:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077455.443657:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617391232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077455.443672:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617390720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077455.443686:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077455.443691:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077455.443695:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617390720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077455.443718:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077455.443743:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077455.443757:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077455/real 1769077455] req@ffff9b0352a1a680 x1855007617390848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077455.443771:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617390848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077455.443780:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077455.443784:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077455.443788:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077455.443795:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077455.443816:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077455.443838:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077455/real 1769077455] req@ffff9b0352a18a80 x1855007617390976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077455.443851:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617390976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077455.443868:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077455.443871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077455.443875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617390976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077455.443882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077455.443924:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077455.443936:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077455/real 1769077455] req@ffff9b0352a1b480 x1855007617391104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077455.443949:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617391104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077455.443958:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077455.443962:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077455.443965:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077455.443972:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077455.443990:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077455.444012:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077455/real 1769077455] req@ffff9b0352a1ad80 x1855007617391232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077455.444024:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617391232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077455.444032:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077455.444036:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077455.444040:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077455.955334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077456.339298:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077456.339300:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077456.339369:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077456.979380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077457.363329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077457.363330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077457.363413:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077458.003353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077458.387294:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077458.387308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077458.387379:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077459.027353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077459.411344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077459.411344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077459.411377:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077460.051341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077460.435316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077460.435316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077460.435377:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077460.563384:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077460.563392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077460.563396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077460.563405:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4449 00000100:00080000:1.0:1769077460.563412:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077460.563418:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077460.563447:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617391360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077460.563470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077460.563473:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077460.563474:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077460.563477:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4449 00000100:00080000:1.0:1769077460.563480:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077460.563482:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077460.563490:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077460.563500:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617391488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077460.563506:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617391360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077460.563511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077460.563514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077460.563516:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077460.563519:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4449 00000100:00080000:1.0:1769077460.563522:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077460.563525:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077460.563530:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077460.563540:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617391616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077460.563548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077460.563550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077460.563552:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077460.563555:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4449 00000100:00080000:1.0:1769077460.563564:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077460.563567:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077460.563571:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077460.563573:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077460.563581:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617391744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077460.563588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077460.563590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077460.563591:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077460.563594:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4449 00000100:00080000:1.0:1769077460.563597:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077460.563599:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077460.563602:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077460.563609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617391872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077460.563619:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077460/real 1769077460] req@ffff9b0343deed80 x1855007617391360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077460.563709:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617391360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077460.563717:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077460.563721:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077460.563723:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617391360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077460.563743:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077460.563873:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077460.563892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077460/real 1769077460] req@ffff9b0343deca80 x1855007617391488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077460.563963:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617391488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077460.563971:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077460.563973:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077460.563976:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077460.564002:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077460.564020:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077460.564029:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077460/real 1769077460] req@ffff9b0343deea00 x1855007617391616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077460.564037:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617391616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077460.564043:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077460.564045:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077460.564047:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077460.564050:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077460.564061:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077460.564067:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077460/real 1769077460] req@ffff9b0343def800 x1855007617391744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077460.564074:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617391744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077460.564079:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077460.564081:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077460.564083:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077460.564087:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077460.564097:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077460.564102:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077460/real 1769077460] req@ffff9b0343ded880 x1855007617391872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077460.564109:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617391872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077460.564114:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077460.564116:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077460.564118:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617391872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077461.075348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077461.459360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077461.459361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077461.459483:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077462.099378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077462.483359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077462.483371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077462.483434:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077463.123354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077463.507359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077463.507359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077463.507422:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077464.147315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077464.531354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077464.531372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077464.531380:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077465.171336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077465.555330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077465.555330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077465.555396:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077465.683358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077465.683369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077465.683375:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077465.683387:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4454 00000100:00080000:3.0:1769077465.683396:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077465.683405:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077465.683443:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617392000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077465.683476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077465.683483:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077465.683485:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077465.683490:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4454 00000100:00080000:3.0:1769077465.683494:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077465.683499:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077465.683508:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077465.683521:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617392128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077465.683534:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617392000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077465.683537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077465.683545:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077465.683547:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077465.683552:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4454 00000100:00080000:3.0:1769077465.683557:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077465.683561:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077465.683567:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077465.683580:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617392256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077465.683589:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077465.683594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077465.683596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077465.683600:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4454 00000100:00080000:3.0:1769077465.683605:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077465.683609:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077465.683616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769077465.683617:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077465.683632:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617392384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077465.683644:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077465.683647:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077465.683649:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077465.683654:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4454 00000100:00080000:3.0:1769077465.683659:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077465.683666:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077465/real 1769077465] req@ffff9b0352a18000 x1855007617392000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077465.683666:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077465.683675:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077465.683695:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617392512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077465.683704:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617392000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077465.683719:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077465.683725:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077465.683731:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617392000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077465.683758:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077465.683782:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077465.683795:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077465/real 1769077465] req@ffff9b0352a18380 x1855007617392128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077465.683809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617392128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077465.683819:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077465.683823:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077465.683826:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077465.683832:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077465.683851:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077465.683860:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077465/real 1769077465] req@ffff9b0352a19c00 x1855007617392256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077465.683872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617392256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077465.683881:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077465.683884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077465.683887:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077465.683927:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077465.683951:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077465.683964:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077465/real 1769077465] req@ffff9b0352a1b800 x1855007617392384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077465.683977:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617392384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077465.683986:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077465.683989:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077465.683993:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077465.683999:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077465.684016:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077465.684025:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077465/real 1769077465] req@ffff9b0352a19500 x1855007617392512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077465.684044:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617392512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077465.684052:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077465.684055:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077465.684058:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077466.195306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077466.579346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077466.579347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077466.579416:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077467.219355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077467.603315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077467.603325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077467.603412:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077468.243322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077468.627394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077468.627394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077468.627481:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077469.267344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077469.651334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077469.651336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077469.651367:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077470.291312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077470.675390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077470.675431:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077470.675499:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077470.803343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077470.803353:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077470.803358:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077470.803368:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4459 00000100:00080000:0.0:1769077470.803379:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077470.803386:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077470.803429:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617392640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077470.803451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077470.803454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077470.803455:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077470.803459:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4459 00000100:00080000:0.0:1769077470.803462:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077470.803465:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077470.803473:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077470.803491:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617392640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077470.803493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617392768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077470.803508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077470.803510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077470.803512:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077470.803515:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4459 00000100:00080000:0.0:1769077470.803517:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077470.803520:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077470.803523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077470.803531:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617392896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077470.803538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077470.803545:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077470.803546:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077470.803548:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4459 00000100:00080000:0.0:1769077470.803551:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077470.803553:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077470.803557:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077470.803566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617393024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077470.803573:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077470.803581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077470.803583:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077470.803585:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077470.803589:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4459 00000100:00080000:0.0:1769077470.803599:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077470.803602:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077470.803605:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077470.803612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617393152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077470.803628:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077470/real 1769077470] req@ffff9b0365a38a80 x1855007617392640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077470.803651:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617392640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077470.803662:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077470.803665:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077470.803669:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617392640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077470.803689:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077470.803708:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077470.803718:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077470/real 1769077470] req@ffff9b0365a39c00 x1855007617392768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077470.803727:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617392768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077470.803732:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077470.803735:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077470.803737:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077470.803741:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077470.803752:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077470.803758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077470/real 1769077470] req@ffff9b0365a38e00 x1855007617392896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077470.803765:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617392896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077470.803770:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077470.803772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077470.803774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617392896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077470.803778:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077470.803788:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077470.803794:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077470/real 1769077470] req@ffff9b0365a3b480 x1855007617393024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077470.803801:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617393024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077470.803830:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077470.803832:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077470.803835:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077470.803839:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077470.803854:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077470.803861:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077470/real 1769077470] req@ffff9b0365a3aa00 x1855007617393152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077470.803869:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617393152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077470.803874:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077470.803876:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077470.803878:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077471.315340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077471.699359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077471.699360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077471.699370:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077472.339316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077472.723336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077472.723336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077472.723390:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077473.363354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077473.747319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077473.747372:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077473.747437:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077474.387346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077474.771344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077474.771345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077474.771397:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077475.411338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077475.795303:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077475.795310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077475.795368:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077475.923353:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077475.923364:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077475.923370:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077475.923381:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4465 00000100:00080000:1.0:1769077475.923390:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077475.923398:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077475.923434:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617393280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077475.923461:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077475.923465:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077475.923467:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077475.923471:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4465 00000100:00080000:1.0:1769077475.923475:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077475.923480:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077475.923488:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077475.923499:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617393408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077475.923508:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077475.923512:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077475.923515:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077475.923519:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4465 00000100:00080000:1.0:1769077475.923523:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077475.923527:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077475.923532:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077475.923541:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617393280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077475.923545:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617393536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077475.923568:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077475.923571:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077475.923573:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077475.923578:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4465 00000100:00080000:1.0:1769077475.923581:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077475.923585:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077475.923591:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077475.923602:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617393664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077475.923610:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1769077475.923615:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077475.923616:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077475.923620:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077475.923626:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4465 00000100:00080000:1.0:1769077475.923630:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077475.923634:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077475.923639:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077475.923651:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617393792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077475.923657:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077475/real 1769077475] req@ffff9b0343dedf80 x1855007617393280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077475.923683:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617393280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077475.923691:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077475.923695:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077475.923697:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617393280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077475.923713:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077475.923755:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077475.923764:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077475/real 1769077475] req@ffff9b0343def100 x1855007617393408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077475.923773:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617393408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077475.923779:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077475.923781:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077475.923784:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077475.923802:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077475.923813:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077475.923819:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077475/real 1769077475] req@ffff9b0343dee300 x1855007617393536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077475.923826:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617393536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077475.923836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077475.923838:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077475.923840:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077475.923843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077475.923852:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077475.923857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077475/real 1769077475] req@ffff9b0343dec380 x1855007617393664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077475.923864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617393664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077475.923870:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077475.923871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077475.923873:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077475.923876:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077475.923885:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077475.923890:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077475/real 1769077475] req@ffff9b0343ded180 x1855007617393792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077475.923897:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617393792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077475.923902:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077475.923903:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077475.923906:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617393792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077476.435348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077476.819344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077476.819344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077476.819370:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077477.459332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077477.843358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077477.843358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077477.843470:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077478.483338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077478.867354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077478.867354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077478.867478:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077479.507344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077479.891311:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077479.891346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077479.891409:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077480.531409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077480.915342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077480.915342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077480.915382:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077481.043333:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.043342:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077481.043346:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.043355:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4470 00000100:00080000:3.0:1769077481.043362:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.043367:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077481.043405:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617393920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077481.043434:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.043438:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077481.043440:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.043444:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4470 00000100:00080000:3.0:1769077481.043448:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.043452:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077481.043462:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077481.043478:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617394048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077481.043486:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617393920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077481.043494:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.043498:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077481.043500:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.043506:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4470 00000100:00080000:3.0:1769077481.043511:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.043516:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077481.043521:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077481.043535:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617394176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077481.043543:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.043545:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077481.043546:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.043549:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4470 00000100:00080000:3.0:1769077481.043552:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.043555:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769077481.043560:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769077481.043562:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077481.043572:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617394304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077481.043579:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077481.043581:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077481.043583:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077481.043586:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4470 00000100:00080000:3.0:1769077481.043589:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077481.043592:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077481.043595:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077481.043603:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617394432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077481.043619:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff9b0352a19500 x1855007617393920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077481.043656:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617393920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077481.043668:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077481.043674:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077481.043678:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617393920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077481.043700:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077481.043720:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077481.043729:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff9b0352a1b800 x1855007617394048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077481.043738:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617394048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077481.043748:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077481.043751:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077481.043753:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077481.043757:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077481.043768:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077481.043778:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff9b0352a19c00 x1855007617394176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077481.043790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617394176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077481.043797:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077481.043801:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077481.043804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077481.043811:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077481.043832:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077481.043841:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff9b0352a18380 x1855007617394304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077481.043850:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617394304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077481.043855:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077481.043857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077481.043860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077481.043864:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077481.043875:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077481.043885:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077481/real 1769077481] req@ffff9b0352a18000 x1855007617394432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077481.043897:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617394432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077481.043905:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077481.043908:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077481.043912:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077481.555333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077481.939296:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077481.939296:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077481.939336:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077482.579334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077482.963326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077482.963328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077482.963393:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077483.603361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077483.987334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077483.987334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077483.987466:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077484.627354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077485.011346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077485.011347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077485.011411:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077485.651341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077486.035372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077486.035384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077486.035446:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077486.163363:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077486.163376:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077486.163380:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077486.163391:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4475 00000100:00080000:1.0:1769077486.163400:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077486.163408:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077486.163445:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617394560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077486.163477:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077486.163482:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077486.163484:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077486.163489:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4475 00000100:00080000:1.0:1769077486.163493:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077486.163497:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077486.163507:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077486.163520:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617394688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077486.163539:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077486.163540:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617394560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077486.163567:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077486.163571:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077486.163579:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4475 00000100:00080000:1.0:1769077486.163583:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077486.163589:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077486.163596:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077486.163613:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617394816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077486.163624:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077486.163630:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077486.163630:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077486.163634:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077486.163641:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4475 00000100:00080000:1.0:1769077486.163647:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077486.163652:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077486.163667:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077486.163669:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077486/real 1769077486] req@ffff9b0343dec000 x1855007617394560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077486.163684:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617394944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077486.163688:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617394560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077486.163716:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077486.163717:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077486.163723:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769077486.163725:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077486.163728:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077486.163729:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617394560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077486.163739:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4475 00000100:00080000:1.0:1769077486.163745:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077486.163749:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077486.163754:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394688:192.168.204.160@tcp:38:kworker.0 00000008:00100000:1.0:1769077486.163757:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077486.163775:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617395072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077486.163778:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077486.163790:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077486/real 1769077486] req@ffff9b0343dec700 x1855007617394688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077486.163804:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617394688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077486.163811:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077486.163814:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077486.163817:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077486.163822:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077486.163834:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077486.163839:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077486/real 1769077486] req@ffff9b0343ded180 x1855007617394816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077486.163847:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617394816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077486.163852:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077486.163854:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077486.163856:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077486.163860:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077486.163870:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077486.163875:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077486/real 1769077486] req@ffff9b0343dec380 x1855007617394944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077486.163883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617394944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077486.163888:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077486.163889:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077486.163891:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617394944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077486.163899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077486.163908:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077486.163914:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077486/real 1769077486] req@ffff9b0343dee300 x1855007617395072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077486.163920:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617395072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077486.163925:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077486.163927:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077486.163929:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077486.675346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077487.059331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077487.059331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077487.059355:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077487.699324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077488.083346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077488.083346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077488.083377:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077488.723346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077489.107342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077489.107343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077489.107437:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077489.747298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077490.131350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077490.131350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077490.131448:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077490.771336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077491.155322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077491.155322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077491.155381:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077491.283340:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077491.283351:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077491.283356:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077491.283367:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4480 00000100:00080000:1.0:1769077491.283377:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077491.283383:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077491.283427:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617395200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077491.283456:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077491.283460:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077491.283492:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077491.283498:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4480 00000100:00080000:1.0:1769077491.283503:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077491.283507:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077491.283513:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617395200:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769077491.283518:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077491.283536:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617395328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077491.283548:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077491.283552:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077491.283554:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077491.283558:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4480 00000100:00080000:1.0:1769077491.283571:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077491.283575:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769077491.283582:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769077491.283583:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077491.283598:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617395456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077491.283607:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077491.283611:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077491.283613:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077491.283618:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4480 00000100:00100000:0.0:1769077491.283622:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077491/real 1769077491] req@ffff9b0343def100 x1855007617395200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077491.283624:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077491.283632:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077491.283640:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077491.283645:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617395200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077491.283654:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769077491.283656:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617395584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077491.283660:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077491.283667:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617395200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077491.283679:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077491.283683:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077491.283686:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077491.283692:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4480 00000100:00100000:0.0:1769077491.283695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395328:192.168.204.160@tcp:38:kworker.0 00000100:00080000:1.0:1769077491.283698:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077491.283704:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077491.283709:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077491.283721:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617395712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077491.283725:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077491.283747:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077491/real 1769077491] req@ffff9b0343dedf80 x1855007617395328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077491.283763:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617395328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077491.283769:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077491.283772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077491.283774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077491.283779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077491.283798:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077491.283808:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077491/real 1769077491] req@ffff9b0343ded880 x1855007617395456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077491.283829:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617395456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077491.283838:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077491.283841:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077491.283845:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077491.283852:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077491.283873:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077491.283884:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077491/real 1769077491] req@ffff9b0343def800 x1855007617395584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077491.283895:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617395584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077491.283908:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077491.283911:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077491.283913:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077491.283925:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077491.283939:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077491.283946:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077491/real 1769077491] req@ffff9b0343deea00 x1855007617395712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077491.283953:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617395712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077491.283962:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077491.283964:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077491.283966:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077491.795302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077492.179398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077492.179429:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077492.179572:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077492.819339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077493.203397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077493.203444:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077493.203937:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077493.843289:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077494.227343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077494.227531:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077494.227597:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077494.867348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077495.251429:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077495.251615:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077495.251830:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077495.891360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077496.275760:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077496.275852:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077496.276548:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077496.403418:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077496.403427:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077496.403431:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077496.403446:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4485 00000100:00080000:2.0:1769077496.403456:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077496.403464:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077496.403499:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617395840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077496.403529:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077496.403533:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077496.403536:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077496.403541:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4485 00000100:00080000:2.0:1769077496.403546:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077496.403550:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077496.403560:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077496.403567:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617395840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077496.403579:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617395968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077496.403593:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077496.403597:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077496.403600:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077496.403606:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4485 00000100:00080000:2.0:1769077496.403618:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077496.403624:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077496.403629:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769077496.403631:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077496.403647:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617396096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077496.403658:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077496.403662:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077496.403664:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077496.403667:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4485 00000100:00080000:2.0:1769077496.403671:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077496.403675:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077496.403681:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077496.403693:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617396224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077496.403705:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077496.403708:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077496.403710:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077496.403715:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4485 00000100:00080000:2.0:1769077496.403719:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077496.403722:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077496/real 1769077496] req@ffff9b03482c6d80 x1855007617395840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077496.403727:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077496.403736:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077496.403748:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617395840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077496.403753:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617396352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077496.403759:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077496.403769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077496.403775:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617395840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077496.403797:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077496.403814:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077496.403943:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077496/real 1769077496] req@ffff9b03482c6680 x1855007617395968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077496.403956:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617395968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077496.403963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077496.403966:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077496.403968:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617395968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077496.403974:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077496.404066:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077496.404077:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077496/real 1769077496] req@ffff9b03482c7100 x1855007617396096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077496.404086:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617396096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077496.404091:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077496.404094:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077496.404096:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077496.404100:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077496.404113:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077496.404119:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077496/real 1769077496] req@ffff9b03482c4000 x1855007617396224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077496.404126:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617396224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077496.404136:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077496.404138:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077496.404140:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077496.404144:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077496.404153:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077496.404158:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077496/real 1769077496] req@ffff9b03482c6a00 x1855007617396352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077496.404165:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617396352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077496.404171:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077496.404172:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077496.404174:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077496.915330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077497.299454:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077497.299573:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077497.299790:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077497.939391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077498.323390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077498.323394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077498.323457:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077498.963393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077499.347317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077499.347318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077499.347398:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077499.987386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077500.371429:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077500.371430:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077500.371519:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077501.011396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077501.395375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077501.395380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077501.395484:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077501.523382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077501.523394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077501.523399:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077501.523408:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4490 00000100:00080000:3.0:1769077501.523416:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077501.523423:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077501.523462:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617396480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077501.523492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077501.523495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077501.523496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077501.523500:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4490 00000100:00080000:3.0:1769077501.523504:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077501.523508:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077501.523519:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077501.523536:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617396608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077501.523548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077501.523552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077501.523554:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077501.523560:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4490 00000100:00080000:3.0:1769077501.523563:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077501.523572:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077501.523575:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617396480:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769077501.523578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077501.523600:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617396736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077501.523613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077501.523618:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077501.523620:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077501.523624:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4490 00000100:00080000:3.0:1769077501.523628:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077501.523633:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077501.523639:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077501.523653:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617396864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077501.523674:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077501.523676:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077501.523680:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077501.523682:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077501.523689:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4490 00000100:00080000:3.0:1769077501.523694:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077501.523699:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077501.523706:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077501.523719:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617396992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077501.523745:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077501/real 1769077501] req@ffff9b0352a1ad80 x1855007617396480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077501.523764:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617396480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077501.523773:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077501.523780:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077501.523785:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617396480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077501.523794:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077501.523821:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077501.523836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077501/real 1769077501] req@ffff9b0352a1b480 x1855007617396608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077501.523851:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617396608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077501.523859:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077501.523862:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077501.523864:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077501.523899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077501.523916:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077501.523925:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077501/real 1769077501] req@ffff9b0352a18a80 x1855007617396736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077501.523933:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617396736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077501.523939:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077501.523941:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077501.523944:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077501.523948:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077501.523960:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077501.523973:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077501/real 1769077501] req@ffff9b0352a1a680 x1855007617396864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077501.523982:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617396864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077501.523987:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077501.523990:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077501.523992:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077501.523996:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077501.524007:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077501.524013:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077501/real 1769077501] req@ffff9b0352a1b100 x1855007617396992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077501.524021:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617396992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077501.524026:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077501.524028:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077501.524030:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617396992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077502.035390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077502.419393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077502.419443:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077502.419448:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077503.059400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077503.443409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077503.443413:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077503.443474:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077504.083418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077504.467374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077504.467376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077504.467452:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077505.107386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077505.491361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077505.491365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077505.491451:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077506.131340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077506.515334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077506.515360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077506.515413:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077506.643393:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077506.643407:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077506.643413:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077506.643427:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4495 00000100:00080000:2.0:1769077506.643440:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077506.643449:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077506.643497:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617397120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077506.643539:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077506.643543:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077506.643545:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077506.643552:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4495 00000100:00080000:2.0:1769077506.643557:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077506.643562:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077506.643573:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077506.643585:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617397248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077506.643598:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077506.643603:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077506.643606:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077506.643613:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617397120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077506.643614:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4495 00000100:00080000:2.0:1769077506.643623:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077506.643634:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077506.643647:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077506.643667:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617397376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077506.643679:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077506.643683:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077506.643685:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077506.643690:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4495 00000100:00080000:2.0:1769077506.643694:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077506.643697:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077506.643703:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077506.643714:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617397504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077506.643724:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077506.643738:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077506.643743:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077506.643745:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077506.643751:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4495 00000100:00080000:2.0:1769077506.643755:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077506.643759:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077506.643765:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077506.643782:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617397632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077506.643786:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077506/real 1769077506] req@ffff9b03482c4700 x1855007617397120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077506.643829:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617397120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077506.643844:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077506.643851:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077506.643856:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617397120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077506.643883:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617397248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077506.643911:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077506.643925:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077506/real 1769077506] req@ffff9b03482c4e00 x1855007617397248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077506.643939:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617397248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077506.643948:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077506.643953:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077506.643957:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617397248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077506.643965:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617397376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077506.643987:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077506.643999:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077506/real 1769077506] req@ffff9b03482c7b80 x1855007617397376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077506.644021:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617397376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077506.644030:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077506.644033:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077506.644037:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617397376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077506.644044:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617397504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077506.644067:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077506.644078:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077506/real 1769077506] req@ffff9b03482c5f80 x1855007617397504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077506.644092:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617397504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077506.644100:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077506.644104:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077506.644108:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617397504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077506.644115:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617397632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077506.644135:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077506.644146:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077506/real 1769077506] req@ffff9b03482c5c00 x1855007617397632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077506.644159:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617397632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077506.644175:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077506.644178:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077506.644182:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617397632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077507.155302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077507.539327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077507.539327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077507.539367:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077508.179348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077508.563329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077508.563347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077508.563405:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077509.203316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077509.587333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077509.587378:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077509.587380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077510.227354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077510.611349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077510.611351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077510.611467:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077511.251299:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077511.635342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077511.635370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077511.635401:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077511.763370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077511.763385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077511.763391:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077511.763405:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4500 00000100:00080000:3.0:1769077511.763418:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077511.763427:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077511.763481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617397760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077511.763510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077511.763513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077511.763515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077511.763519:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4500 00000100:00080000:3.0:1769077511.763522:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077511.763525:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077511.763535:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077511.763546:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617397888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077511.763555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077511.763558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077511.763561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077511.763568:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4500 00000100:00080000:3.0:1769077511.763571:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077511.763572:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617397760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077511.763578:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077511.763589:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077511.763608:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617398016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077511.763614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077511.763616:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077511.763618:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077511.763621:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4500 00000100:00080000:3.0:1769077511.763626:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077511.763630:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077511.763634:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077511.763640:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617398144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077511.763646:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077511.763650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077511.763651:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077511.763654:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4500 00000100:00080000:3.0:1769077511.763657:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077511.763659:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077511.763662:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077511.763675:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617398272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077511.763701:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617397888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077511.763724:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077511.763741:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077511.763762:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077512.275348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077512.659343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077512.659356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077512.659400:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077513.299310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077513.683348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077513.683365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077513.683422:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077514.323320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077514.707315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077514.707324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077514.707389:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769077514.835532:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769077514.835582:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077514.835601:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077514.835648:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077514.835660:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077514.835674:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077514.835685:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769077514.835736:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077511/real 1769077514] req@ffff9b0352a1aa00 x1855007617397888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077514.835770:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617397888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077514.835796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077514.835804:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077514.835809:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617397888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077514.835819:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077511/real 1769077514] req@ffff9b0352a18700 x1855007617398016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077514.835835:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617398016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077514.835845:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077514.835849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077514.835852:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077514.835859:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077511/real 1769077514] req@ffff9b0352a19f80 x1855007617398144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077514.835872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617398144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077514.835881:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077514.835885:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077514.835889:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077514.835897:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077511/real 1769077514] req@ffff9b0352a1a300 x1855007617398272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077514.835911:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617398272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077514.835921:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077514.835925:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077514.835929:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077514.835936:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077511/real 1769077514] req@ffff9b0352a18e00 x1855007617397760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077514.835950:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617397760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077514.835958:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077514.835963:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077514.835966:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617397760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077515.347344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077515.731347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077515.731347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077515.731424:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077516.371335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077516.755313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077516.755346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077516.755403:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077516.883364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077516.883378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077516.883383:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077516.883397:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4506 00000100:00080000:3.0:1769077516.883408:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077516.883416:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077516.883458:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617398400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077516.883486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077516.883489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077516.883491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077516.883494:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4506 00000100:00080000:3.0:1769077516.883497:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077516.883499:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077516.883508:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077516.883519:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617398528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077516.883530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077516.883534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077516.883536:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077516.883541:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4506 00000100:00080000:3.0:1769077516.883545:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077516.883551:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077516.883553:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617398400:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769077516.883558:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077516.883573:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617398656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077516.883585:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077516.883589:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077516.883592:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077516.883598:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4506 00000100:00080000:3.0:1769077516.883604:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077516.883608:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077516.883614:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077516.883626:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617398784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077516.883637:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077516.883641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077516.883643:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077516.883647:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4506 00000100:00080000:3.0:1769077516.883651:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077516.883655:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077516.883661:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769077516.883665:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077516.883677:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617398912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077516.883721:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077516/real 1769077516] req@ffff9b0352a18e00 x1855007617398400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077516.883745:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617398400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077516.883758:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077516.883763:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077516.883768:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617398400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077516.883781:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077516.883807:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077516.883828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077516/real 1769077516] req@ffff9b0352a1a300 x1855007617398528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077516.883844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617398528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077516.883854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077516.883857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077516.883861:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077516.883893:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077516.883916:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077516.883927:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077516/real 1769077516] req@ffff9b0352a19f80 x1855007617398656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077516.883940:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617398656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077516.883949:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077516.883953:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077516.883956:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077516.883963:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077516.883985:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077516.883996:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077516/real 1769077516] req@ffff9b0352a18700 x1855007617398784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077516.884010:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617398784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077516.884026:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077516.884043:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077516.884047:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077516.884054:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077516.884077:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077516.884089:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077516/real 1769077516] req@ffff9b0352a1aa00 x1855007617398912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077516.884102:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617398912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077516.884111:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077516.884114:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077516.884118:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617398912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077517.395370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077517.779399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077517.779399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077517.779495:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077518.419320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077518.803307:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077518.803308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077518.803396:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077519.443347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077519.827335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077519.827335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077519.827407:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077520.467331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077520.851294:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077520.851311:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077520.851359:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077521.491329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077521.875361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077521.875361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077521.875456:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077522.003334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.003348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077522.003352:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.003364:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4511 00000100:00080000:1.0:1769077522.003374:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.003382:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077522.003416:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617399040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077522.003438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.003441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077522.003443:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.003446:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4511 00000100:00080000:1.0:1769077522.003449:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.003452:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077522.003460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077522.003470:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617399168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077522.003478:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.003481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077522.003483:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077522.003487:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617399040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077522.003488:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4511 00000100:00080000:1.0:1769077522.003493:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.003495:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077522.003503:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077522.003512:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617399296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077522.003519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.003521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077522.003522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.003525:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4511 00000100:00080000:1.0:1769077522.003528:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077522.003530:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077522.003533:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077522.003540:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617399424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077522.003545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077522.003548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077522.003549:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077522.003551:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4511 00000100:00080000:1.0:1769077522.003554:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077522.003556:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077522.003560:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077522.003564:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077522.003572:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617399552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077522.003596:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9b0343deca80 x1855007617399040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077522.003612:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617399040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077522.003619:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077522.003622:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077522.003625:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617399040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077522.003643:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077522.003657:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077522.003665:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9b0343deed80 x1855007617399168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077522.003673:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617399168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077522.003679:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077522.003681:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077522.003683:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077522.003687:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077522.003697:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077522.003702:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9b0343dee680 x1855007617399296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077522.003709:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617399296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077522.003719:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077522.003720:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077522.003723:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077522.003726:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077522.003735:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077522.003740:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9b0343defb80 x1855007617399424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077522.003747:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617399424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077522.003751:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077522.003753:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077522.003755:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077522.003758:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077522.003767:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077522.003772:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077522/real 1769077522] req@ffff9b0343dece00 x1855007617399552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077522.003778:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617399552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077522.003783:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077522.003785:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077522.003787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077522.515390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077522.899361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077522.899361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077522.899427:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077523.539372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077523.923352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077523.923352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077523.923421:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077524.563353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077524.947373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077524.947373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077524.947444:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077525.587394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077525.971387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077525.971387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077525.971489:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077526.611388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077526.995360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077526.995360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077526.995444:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077527.123373:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077527.123391:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077527.123398:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077527.123409:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4516 00000100:00080000:3.0:1769077527.123421:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077527.123428:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077527.123473:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617399680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077527.123503:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077527.123506:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077527.123508:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077527.123511:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4516 00000100:00080000:3.0:1769077527.123513:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077527.123516:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077527.123526:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077527.123542:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617399808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077527.123554:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077527.123557:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617399680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077527.123560:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077527.123566:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077527.123574:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4516 00000100:00080000:3.0:1769077527.123579:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077527.123584:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077527.123591:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077527.123606:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617399936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077527.123616:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077527.123636:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077527.123636:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077527.123642:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077527.123650:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4516 00000100:00080000:3.0:1769077527.123656:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077527.123662:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077527.123669:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077527.123685:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617400064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077527.123687:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077527/real 1769077527] req@ffff9b0352a1b100 x1855007617399680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077527.123717:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077527.123722:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769077527.123723:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617399680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077527.123727:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077527.123738:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077527.123739:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4516 00000100:00080000:0.0:1769077527.123745:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769077527.123747:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077527.123750:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617399680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077527.123756:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077527.123763:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077527.123776:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077527.123782:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617400192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077527.123795:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077527.123808:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077527/real 1769077527] req@ffff9b0352a1a680 x1855007617399808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077527.123817:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617399808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077527.123825:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077527.123828:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077527.123831:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077527.123836:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077527.123847:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077527.123853:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077527/real 1769077527] req@ffff9b0352a18a80 x1855007617399936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077527.123876:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617399936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077527.123896:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077527.123900:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077527.123904:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617399936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077527.123911:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077527.123935:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077527.123947:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077527/real 1769077527] req@ffff9b0352a1b480 x1855007617400064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077527.123960:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617400064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077527.123969:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077527.123971:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077527.123973:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077527.123994:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077527.124010:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077527.124018:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077527/real 1769077527] req@ffff9b0352a1ad80 x1855007617400192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077527.124026:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617400192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077527.124032:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077527.124035:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077527.124037:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077527.635394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077528.019352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077528.019352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077528.019455:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077528.659346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077529.043395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077529.043395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077529.043467:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077529.683384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077530.067359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077530.067368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077530.067432:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077530.707332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077531.091417:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077531.091418:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077531.091462:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077531.731372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077532.115330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077532.115330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077532.115401:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077532.243356:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077532.243369:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077532.243373:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077532.243384:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4521 00000100:00080000:0.0:1769077532.243394:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077532.243400:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077532.243432:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617400320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077532.243460:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077532.243465:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077532.243466:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077532.243471:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4521 00000100:00080000:0.0:1769077532.243475:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077532.243481:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077532.243491:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077532.243507:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617400448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077532.243507:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617400320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077532.243523:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077532.243526:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077532.243528:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077532.243531:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4521 00000100:00080000:0.0:1769077532.243535:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077532.243538:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077532.243542:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077532.243551:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617400576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077532.243557:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077532.243560:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077532.243561:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077532.243563:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4521 00000100:00080000:0.0:1769077532.243565:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077532.243568:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077532.243571:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077532.243579:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617400704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077532.243585:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077532.243592:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077532.243595:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077532.243597:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077532.243601:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4521 00000100:00080000:0.0:1769077532.243605:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077532.243608:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077532.243611:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077532.243618:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617400832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077532.243626:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077532/real 1769077532] req@ffff9b0365a3b100 x1855007617400320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077532.243643:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617400320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077532.243652:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077532.243655:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077532.243658:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617400320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077532.243676:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077532.243694:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077532.243703:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077532/real 1769077532] req@ffff9b0365a39500 x1855007617400448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077532.243716:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617400448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077532.243722:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077532.243724:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077532.243727:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077532.243732:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077532.243744:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077532.243751:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077532/real 1769077532] req@ffff9b0365a3bb80 x1855007617400576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077532.243760:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617400576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077532.243765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077532.243768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077532.243770:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077532.243793:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077532.243822:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077532.243836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077532/real 1769077532] req@ffff9b0365a3ad80 x1855007617400704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077532.243849:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617400704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077532.243858:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077532.243862:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077532.243866:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077532.243873:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077532.243894:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077532.243969:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077532/real 1769077532] req@ffff9b0365a39180 x1855007617400832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077532.243985:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617400832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077532.243993:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077532.243996:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077532.244000:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617400832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077532.755367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077533.139385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077533.139386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077533.139421:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077533.779396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077534.163324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077534.163433:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077534.163437:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077534.803320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077535.187347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077535.187395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077535.187457:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077535.827318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077536.211312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077536.211316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077536.211379:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077536.851345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077537.235342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077537.235342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077537.235405:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077537.363323:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077537.363332:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077537.363336:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077537.363344:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4526 00000100:00080000:3.0:1769077537.363351:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077537.363357:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077537.363390:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617400960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077537.363423:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077537.363427:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077537.363429:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077537.363434:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4526 00000100:00080000:3.0:1769077537.363437:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077537.363442:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077537.363451:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077537.363465:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617401088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077537.363476:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617400960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077537.363479:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077537.363497:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077537.363499:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077537.363504:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4526 00000100:00080000:3.0:1769077537.363509:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077537.363513:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077537.363519:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077537.363531:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617401216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077537.363541:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077537.363543:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077537.363548:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077537.363551:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077537.363556:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4526 00000100:00080000:3.0:1769077537.363560:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077537.363565:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077537.363570:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077537.363583:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617401344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077537.363590:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077537/real 1769077537] req@ffff9b0352a18000 x1855007617400960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077537.363595:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077537.363605:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077537.363609:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077537.363615:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4526 00000100:00080000:3.0:1769077537.363619:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077537.363621:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617400960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077537.363626:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077537.363635:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1769077537.363640:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077537.363648:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077537.363650:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617401472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077537.363656:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617400960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077537.363699:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077537.363723:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077537.363736:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077537/real 1769077537] req@ffff9b0352a18380 x1855007617401088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077537.363749:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617401088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077537.363759:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077537.363762:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077537.363766:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077537.363772:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077537.363789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077537.363798:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077537/real 1769077537] req@ffff9b0352a19c00 x1855007617401216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077537.363811:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617401216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077537.363819:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077537.363823:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077537.363826:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077537.363832:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077537.363849:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077537.363858:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077537/real 1769077537] req@ffff9b0352a1b800 x1855007617401344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077537.363878:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617401344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077537.363886:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077537.363890:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077537.363893:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077537.363899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077537.363913:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077537.363922:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077537/real 1769077537] req@ffff9b0352a19500 x1855007617401472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077537.363935:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617401472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077537.363944:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077537.363947:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077537.363951:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077537.875334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077538.259309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077538.259310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077538.259334:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077538.899356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077539.283343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077539.283343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077539.283411:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077539.923339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077540.307340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077540.307340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077540.307404:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077540.947341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077541.331357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077541.331357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077541.331421:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077541.971330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077542.355337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077542.355337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077542.355521:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077542.483381:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077542.483390:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077542.483395:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077542.483403:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4531 00000100:00080000:0.0:1769077542.483419:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077542.483428:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077542.483462:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617401600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077542.483486:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077542.483488:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077542.483490:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077542.483493:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4531 00000100:00080000:0.0:1769077542.483495:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077542.483497:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077542.483504:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077542.483513:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617401728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077542.483520:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077542.483522:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077542.483524:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077542.483526:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4531 00000100:00080000:0.0:1769077542.483528:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077542.483533:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617401600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077542.483534:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077542.483539:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077542.483547:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617401856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077542.483559:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077542.483561:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077542.483563:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077542.483565:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4531 00000100:00080000:0.0:1769077542.483567:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077542.483569:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077542.483572:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077542.483579:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617401984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077542.483585:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077542.483588:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077542.483589:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077542.483592:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4531 00000100:00080000:0.0:1769077542.483594:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077542.483596:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077542.483599:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077542.483622:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077542.483633:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617402112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077542.483670:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077542/real 1769077542] req@ffff9b0365a39180 x1855007617401600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077542.483691:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617401600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077542.483702:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077542.483707:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077542.483711:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617401600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077542.483720:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077542.483745:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077542.483759:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077542/real 1769077542] req@ffff9b0365a3ad80 x1855007617401728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077542.483773:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617401728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077542.483783:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077542.483786:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077542.483790:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077542.483816:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077542.483838:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077542.483850:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077542/real 1769077542] req@ffff9b0365a3bb80 x1855007617401856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077542.483863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617401856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077542.483872:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077542.483876:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077542.483880:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077542.483886:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077542.483921:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077542.483934:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077542/real 1769077542] req@ffff9b0365a39500 x1855007617401984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077542.483947:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617401984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077542.483956:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077542.483960:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077542.483963:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617401984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077542.483969:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617402112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077542.483987:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077542.483997:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077542/real 1769077542] req@ffff9b0365a3b100 x1855007617402112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077542.484009:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617402112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077542.484018:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077542.484021:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077542.484024:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617402112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077542.995411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077543.379333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077543.379333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077543.379359:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077544.019323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077544.403373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077544.403374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077544.403456:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077545.043396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077545.427354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077545.427354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077545.427414:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077546.067338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077546.451360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077546.451378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077546.451434:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077547.091380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077547.475332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077547.475342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077547.475400:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077547.603337:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077547.603345:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077547.603349:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077547.603358:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4536 00000100:00080000:0.0:1769077547.603366:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077547.603373:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077547.603409:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617402240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077547.603441:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077547.603451:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077547.603453:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077547.603463:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4536 00000100:00080000:0.0:1769077547.603468:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077547.603473:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077547.603483:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077547.603486:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617402240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077547.603502:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617402368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077547.603529:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077547.603534:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077547.603536:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077547.603540:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4536 00000100:00080000:0.0:1769077547.603545:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077547.603549:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077547.603556:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769077547.603569:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077547.603572:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617402496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077547.603585:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077547.603588:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077547.603590:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077547.603595:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4536 00000100:00080000:0.0:1769077547.603599:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077547.603604:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077547.603608:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077547/real 1769077547] req@ffff9b0365a3aa00 x1855007617402240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077547.603612:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077547.603630:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617402624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077547.603632:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617402240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077547.603661:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077547.603669:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077547.603669:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077547.603672:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077547.603677:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4536 00000100:00080000:0.0:1769077547.603682:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077547.603686:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:0.0:1769077547.603692:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077547.603693:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617402240:192.168.204.160@tcp:250:kworker.0 00000008:00100000:0.0:1769077547.603704:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077547.603718:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617402752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077547.603721:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617402368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077547.603748:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077547.603758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077547/real 1769077547] req@ffff9b0365a3b480 x1855007617402368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077547.603767:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617402368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077547.603773:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077547.603776:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077547.603778:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617402368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077547.603782:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617402496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077547.603793:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077547.603798:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077547/real 1769077547] req@ffff9b0365a38e00 x1855007617402496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077547.603806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617402496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077547.603811:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077547.603813:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077547.603815:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617402496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077547.603819:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617402624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077547.603829:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077547.603835:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077547/real 1769077547] req@ffff9b0365a39c00 x1855007617402624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077547.603842:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617402624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077547.603847:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077547.603849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077547.603850:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617402624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077547.603860:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617402752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077547.603869:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077547.603875:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077547/real 1769077547] req@ffff9b0365a38a80 x1855007617402752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077547.603882:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617402752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077547.603887:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077547.603889:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077547.603891:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617402752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077548.115328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077548.499342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077548.499342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077548.499357:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077549.139339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077549.523348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077549.523355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077549.523413:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077550.163330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077550.547356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077550.547356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077550.547425:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077551.187397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077551.571334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077551.571334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077551.571405:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077552.211329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077552.595366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077552.595366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077552.595450:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077552.723443:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077552.723456:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077552.723462:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077552.723503:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4541 00000100:00080000:3.0:1769077552.723518:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077552.723527:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077552.723601:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617402880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077552.723635:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077552.723638:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077552.723640:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077552.723644:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4541 00000100:00080000:3.0:1769077552.723647:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077552.723651:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077552.723668:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077552.723697:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617403008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077552.723708:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077552.723711:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077552.723715:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077552.723724:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617402880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077552.723730:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4541 00000100:00080000:3.0:1769077552.723741:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077552.723748:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077552.723753:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077552.723772:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617403136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077552.723781:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077552.723784:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077552.723785:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077552.723788:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4541 00000100:00080000:3.0:1769077552.723794:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077552.723797:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077552.723800:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077552.723810:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617403264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077552.723816:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077552.723826:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077552.723827:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077552.723832:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077552.723838:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4541 00000100:00080000:3.0:1769077552.723844:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077552.723847:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077552.723854:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077552.723879:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617403392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077552.723889:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077552/real 1769077552] req@ffff9b0352a19500 x1855007617402880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077552.723930:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617402880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077552.723945:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077552.723951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077552.723955:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617402880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077552.723981:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077552.724003:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077552.724014:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077552/real 1769077552] req@ffff9b0352a1b800 x1855007617403008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077552.724026:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617403008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077552.724034:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077552.724037:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077552.724041:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077552.724047:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077552.724062:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077552.724070:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077552/real 1769077552] req@ffff9b0352a19c00 x1855007617403136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077552.724081:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617403136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077552.724088:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077552.724090:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077552.724093:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077552.724098:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077552.724123:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077552.724130:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077552/real 1769077552] req@ffff9b0352a18380 x1855007617403264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077552.724140:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617403264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077552.724147:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077552.724150:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077552.724153:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077552.724158:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077552.724183:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077552.724190:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077552/real 1769077552] req@ffff9b0352a18000 x1855007617403392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077552.724207:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617403392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077552.724214:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077552.724216:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077552.724219:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077553.235381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077553.619319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077553.619319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077553.619360:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077554.259360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077554.643340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077554.643350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077554.643419:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077555.283347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077555.667316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077555.667329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077555.667424:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077556.307337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077556.691337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077556.691337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077556.691423:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077557.331381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077557.715342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077557.715342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077557.715435:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077557.843308:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077557.843317:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077557.843321:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077557.843331:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4547 00000100:00080000:3.0:1769077557.843339:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077557.843367:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077557.843399:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617403520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077557.843421:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077557.843424:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077557.843426:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077557.843429:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4547 00000100:00080000:3.0:1769077557.843431:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077557.843434:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077557.843442:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077557.843451:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617403648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077557.843459:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077557.843462:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077557.843463:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077557.843469:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617403520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077557.843469:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4547 00000100:00080000:3.0:1769077557.843474:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077557.843479:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077557.843482:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077557.843490:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617403776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077557.843497:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077557.843499:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077557.843501:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077557.843503:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4547 00000100:00080000:3.0:1769077557.843505:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077557.843507:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077557.843510:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077557.843516:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617403904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077557.843523:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077557.843525:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077557.843526:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077557.843532:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4547 00000800:00000100:0.0:1769077557.843535:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077557.843536:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077557.843541:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077557.843544:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077557.843552:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617404032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077557.843587:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077557/real 1769077557] req@ffff9b0352a1ad80 x1855007617403520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077557.843605:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617403520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077557.843612:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077557.843616:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077557.843619:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617403520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077557.843638:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077557.843667:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077557.843680:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077557/real 1769077557] req@ffff9b0352a1b480 x1855007617403648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077557.843694:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617403648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077557.843711:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077557.843714:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077557.843718:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077557.843724:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077557.843740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077557.843747:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077557/real 1769077557] req@ffff9b0352a18a80 x1855007617403776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077557.843755:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617403776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077557.843760:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077557.843762:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077557.843764:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077557.843768:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077557.843777:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077557.843782:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077557/real 1769077557] req@ffff9b0352a1a680 x1855007617403904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077557.843789:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617403904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077557.843794:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077557.843796:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077557.843798:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617403904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077557.843801:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077557.843809:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077557.843814:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077557/real 1769077557] req@ffff9b0352a1b100 x1855007617404032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077557.843821:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617404032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077557.843825:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077557.843827:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077557.843829:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077558.355327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077558.739398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077558.739419:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077558.739859:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077559.379339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077559.763456:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077559.763523:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077559.763574:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077560.403358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077560.787448:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077560.787448:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077560.787470:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077561.427338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077561.811350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077561.811350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077561.811900:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077562.451348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077562.835374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077562.835466:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077562.835609:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077562.963355:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077562.963364:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077562.963368:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077562.963376:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4552 00000100:00080000:3.0:1769077562.963385:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077562.963391:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077562.963423:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617404160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077562.963451:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077562.963454:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077562.963455:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077562.963458:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4552 00000100:00080000:3.0:1769077562.963461:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077562.963463:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077562.963470:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077562.963479:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617404288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077562.963486:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077562.963488:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077562.963489:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077562.963492:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4552 00000100:00080000:3.0:1769077562.963493:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077562.963495:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077562.963499:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077562.963506:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617404416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077562.963513:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077562.963515:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077562.963516:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077562.963519:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4552 00000100:00080000:3.0:1769077562.963521:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077562.963526:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617404160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077562.963526:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077562.963533:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077562.963541:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617404544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077562.963548:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077562.963550:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077562.963551:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077562.963553:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4552 00000100:00080000:3.0:1769077562.963555:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077562.963558:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077562.963561:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077562.963568:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617404672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077562.963598:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077562.963633:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077562/real 1769077562] req@ffff9b0352a1aa00 x1855007617404160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077562.963646:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617404160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077562.963654:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077562.963657:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077562.963660:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617404160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077562.963666:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077562.963682:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077562.963690:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077562/real 1769077562] req@ffff9b0352a18700 x1855007617404288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077562.963708:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617404288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077562.963713:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077562.963715:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077562.963717:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077562.963721:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077562.963731:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077562.963736:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077562/real 1769077562] req@ffff9b0352a19f80 x1855007617404416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077562.963743:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617404416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077562.963749:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077562.963750:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077562.963752:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077562.963780:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077562.963791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077562.963797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077562/real 1769077562] req@ffff9b0352a1a300 x1855007617404544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077562.963809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617404544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077562.963814:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077562.963816:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077562.963817:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077562.963821:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077562.963830:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077562.963844:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077562/real 1769077562] req@ffff9b0352a18e00 x1855007617404672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077562.963853:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617404672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077562.963858:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077562.963860:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077562.963862:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077563.475330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077563.859364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077563.859411:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077563.859551:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077564.499352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077564.883336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077564.883336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077564.883410:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077565.523318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077565.907378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077565.907404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077565.907437:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077566.547387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077566.931352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077566.931414:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077566.931424:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077567.571309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077567.955398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077567.955398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077567.955570:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077568.083378:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077568.083401:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077568.083407:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077568.083423:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4557 00000100:00080000:3.0:1769077568.083435:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077568.083443:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077568.083483:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617404800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077568.083530:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077568.083534:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077568.083536:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077568.083540:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4557 00000100:00080000:3.0:1769077568.083544:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077568.083548:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077568.083557:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077568.083571:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617404928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077568.083583:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077568.083588:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077568.083591:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077568.083598:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4557 00000100:00080000:3.0:1769077568.083603:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077568.083610:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077568.083620:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077568.083633:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617404800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077568.083647:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617405056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077568.083668:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077568.083673:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077568.083676:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077568.083683:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4557 00000100:00080000:3.0:1769077568.083689:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077568.083695:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077568.083704:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077568.083724:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617405184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077568.083739:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077568.083751:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077568.083755:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769077568.083761:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077568.083767:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4557 00000100:00080000:3.0:1769077568.083780:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077568.083788:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077568.083798:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077568.083818:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617405312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077568.083830:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077568/real 1769077568] req@ffff9b0352a18e00 x1855007617404800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077568.083862:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617404800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077568.083887:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077568.083892:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077568.083897:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617404800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077568.083907:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077568.083931:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077568.083942:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077568/real 1769077568] req@ffff9b0352a1a300 x1855007617404928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077568.083952:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617404928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077568.083963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077568.083967:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077568.083972:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617404928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077568.084027:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077568.084065:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077568.084078:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077568/real 1769077568] req@ffff9b0352a19f80 x1855007617405056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077568.084090:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617405056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077568.084100:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077568.084104:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077568.084111:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077568.084126:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077568.084154:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077568.084190:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077568/real 1769077568] req@ffff9b0352a18700 x1855007617405184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077568.084215:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617405184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077568.084224:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077568.084309:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077568.084332:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077568.084345:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077568.084377:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077568.084395:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077568/real 1769077568] req@ffff9b0352a1aa00 x1855007617405312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077568.084406:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617405312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077568.084415:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077568.084419:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077568.084422:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077568.595336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077568.979346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077568.979346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077568.979414:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077569.619364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077570.003353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077570.003353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077570.003422:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077570.643387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077571.027340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077571.027342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077571.027381:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077571.667397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077572.051423:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077572.051424:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077572.051514:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077572.691379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077573.075334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077573.075354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077573.075430:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077573.203347:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077573.203360:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077573.203364:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077573.203375:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4562 00000100:00080000:3.0:1769077573.203394:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077573.203406:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077573.203447:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617405440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077573.203475:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077573.203478:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077573.203480:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077573.203483:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4562 00000100:00080000:3.0:1769077573.203486:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077573.203489:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077573.203497:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077573.203511:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617405568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077573.203522:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077573.203524:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077573.203526:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077573.203528:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4562 00000100:00080000:3.0:1769077573.203532:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077573.203542:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617405440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077573.203542:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077573.203554:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077573.203563:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617405696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077573.203570:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077573.203573:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077573.203574:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077573.203578:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4562 00000100:00080000:3.0:1769077573.203580:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077573.203582:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077573.203585:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077573.203593:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617405824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077573.203599:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077573.203601:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077573.203603:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077573.203608:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4562 00000100:00080000:3.0:1769077573.203610:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077573.203612:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077573.203615:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077573.203621:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617405952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077573.203636:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077573.203685:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077573/real 1769077573] req@ffff9b0352a1b100 x1855007617405440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077573.203701:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617405440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077573.203711:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077573.203715:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077573.203719:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617405440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077573.203743:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077573.203759:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077573.203768:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077573/real 1769077573] req@ffff9b0352a1a680 x1855007617405568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077573.203776:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617405568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077573.203783:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077573.203785:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077573.203787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077573.203791:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077573.203802:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077573.203807:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077573/real 1769077573] req@ffff9b0352a18a80 x1855007617405696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077573.203815:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617405696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077573.203826:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077573.203828:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077573.203830:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077573.203833:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077573.203844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077573.203849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077573/real 1769077573] req@ffff9b0352a1b480 x1855007617405824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077573.203856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617405824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077573.203862:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077573.203863:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077573.203865:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077573.203869:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077573.203879:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077573.203885:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077573/real 1769077573] req@ffff9b0352a1ad80 x1855007617405952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077573.203892:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617405952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077573.203897:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077573.203899:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077573.203902:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617405952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077573.715341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077574.099330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077574.099330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077574.099372:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077574.739319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077575.123337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077575.123345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077575.123435:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077575.763357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077576.147336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077576.147336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077576.147392:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077576.787328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077577.171299:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077577.171299:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077577.171371:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077577.811326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077578.195396:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077578.195397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077578.195470:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077578.323346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077578.323358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077578.323363:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077578.323373:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4567 00000100:00080000:1.0:1769077578.323384:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077578.323391:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077578.323436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617406080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077578.323464:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077578.323468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077578.323473:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077578.323477:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4567 00000100:00080000:1.0:1769077578.323480:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077578.323484:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077578.323494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077578.323504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617406208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077578.323512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077578.323514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077578.323515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077578.323518:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4567 00000100:00080000:1.0:1769077578.323520:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077578.323525:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077578.323526:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617406080:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769077578.323528:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077578.323541:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617406336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077578.323549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077578.323552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077578.323553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077578.323556:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4567 00000100:00080000:1.0:1769077578.323558:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077578.323560:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077578.323564:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077578.323573:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617406464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077578.323579:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077578.323582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077578.323583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077578.323586:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4567 00000100:00080000:1.0:1769077578.323601:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077578.323604:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077578.323607:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077578.323615:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617406592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077578.323649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077578.323685:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077578.323714:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077578.323745:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077578.835315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077579.219356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077579.219367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077579.219480:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077579.859337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077580.243374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077580.243375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077580.243451:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077580.883358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077581.267347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077581.267348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077581.267396:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769077581.395590:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769077581.395647:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769077581.395671:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077581.395720:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077581.395731:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077581.395741:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077581.395751:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:3.0:1769077581.395769:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077578/real 1769077581] req@ffff9b0343dece00 x1855007617406336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077581.395799:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617406336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077581.395812:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077581.395818:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077581.395823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077581.395831:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077578/real 1769077581] req@ffff9b0343defb80 x1855007617406464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077581.395845:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617406464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077581.395853:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077581.395862:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077581.395866:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077581.395872:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077578/real 1769077581] req@ffff9b0343dee680 x1855007617406592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077581.395890:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617406592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077581.395903:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077581.395907:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077581.395910:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077581.395917:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077578/real 1769077581] req@ffff9b0343def480 x1855007617406080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077581.395929:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617406080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077581.395938:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077581.395941:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077581.395944:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617406080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077581.395964:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077578/real 1769077581] req@ffff9b0343ded500 x1855007617406208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077581.395976:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617406208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077581.395984:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077581.395988:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077581.395991:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077581.907303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077582.291351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077582.291351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077582.291506:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077582.931312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077583.315337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077583.315338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077583.315423:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077583.443356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077583.443366:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077583.443369:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077583.443379:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4572 00000100:00080000:1.0:1769077583.443388:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077583.443394:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077583.443423:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617406720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077583.443456:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077583.443460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077583.443462:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077583.443467:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4572 00000100:00080000:1.0:1769077583.443471:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077583.443474:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077583.443485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077583.443510:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617406848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077583.443526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077583.443529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077583.443531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077583.443535:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4572 00000100:00080000:1.0:1769077583.443539:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077583.443543:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077583.443548:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077583.443559:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617406976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077583.443568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077583.443571:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617406720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077583.443572:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077583.443573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077583.443577:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4572 00000100:00080000:1.0:1769077583.443580:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077583.443584:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077583.443588:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077583.443596:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617407104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077583.443605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077583.443608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077583.443609:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077583.443612:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4572 00000100:00080000:1.0:1769077583.443615:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077583.443618:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077583.443622:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077583.443630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617407232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077583.443632:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077583.443669:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077583/real 1769077583] req@ffff9b0343deed80 x1855007617406720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077583.444814:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617406720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077583.444827:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077583.444832:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077583.444836:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617406720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077583.444847:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077583.444893:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077583.444910:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077583/real 1769077583] req@ffff9b0343deca80 x1855007617406848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077583.444936:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617406848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077583.444944:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077583.444947:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077583.444951:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077583.444976:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077583.444998:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077583.445009:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077583/real 1769077583] req@ffff9b0343deea00 x1855007617406976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077583.445021:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617406976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077583.445035:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077583.445039:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077583.445042:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617406976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077583.445049:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077583.445069:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077583.445080:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077583/real 1769077583] req@ffff9b0343def800 x1855007617407104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077583.445092:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617407104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077583.445101:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077583.445104:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077583.445108:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077583.445114:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077583.445132:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077583.445140:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077583/real 1769077583] req@ffff9b0343ded880 x1855007617407232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077583.445151:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617407232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077583.445159:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077583.445163:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077583.445167:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077583.955968:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077584.339368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077584.339381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077584.339460:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077584.979319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077585.363365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077585.363366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077585.363445:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077586.003303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077586.387334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077586.387349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077586.387413:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077587.027311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077587.411339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077587.411346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077587.411402:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077588.052285:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077588.435335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077588.435361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077588.435390:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077588.563367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077588.563379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077588.563384:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077588.563394:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4577 00000100:00080000:0.0:1769077588.563403:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077588.563409:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077588.563444:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617407360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077588.563473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077588.563477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077588.563479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077588.563484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4577 00000100:00080000:0.0:1769077588.563488:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077588.563492:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077588.563502:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077588.563516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617407488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077588.563528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077588.563534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1769077588.563536:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617407360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077588.563538:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077588.563547:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4577 00000100:00080000:0.0:1769077588.563552:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077588.563557:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077588.563563:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077588.563574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617407616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077588.563581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077588.563584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077588.563585:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077588.563588:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4577 00000100:00080000:0.0:1769077588.563593:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077588.563597:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077588.563603:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769077588.563607:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077588.563624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617407744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077588.563636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077588.563640:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077588/real 1769077588] req@ffff9b0365a38a80 x1855007617407360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077588.563642:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077588.563647:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077588.563655:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4577 00000100:00080000:0.0:1769077588.563661:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077588.563664:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617407360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077588.563668:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077588.563675:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1769077588.563687:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077588.563689:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617407872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077588.563696:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077588.563706:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617407360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077588.563734:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077588.563760:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077588.563775:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077588/real 1769077588] req@ffff9b0365a39c00 x1855007617407488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077588.563803:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617407488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077588.563811:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077588.563814:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077588.563817:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077588.563822:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077588.563851:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077588.563859:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077588/real 1769077588] req@ffff9b0365a38e00 x1855007617407616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077588.563868:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617407616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077588.563873:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077588.563876:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077588.563878:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077588.563882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077588.563893:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077588.563899:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077588/real 1769077588] req@ffff9b0365a3b480 x1855007617407744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077588.563907:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617407744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077588.563911:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077588.563913:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077588.563915:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077588.563918:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077588.563928:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077588.563934:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077588/real 1769077588] req@ffff9b0365a3aa00 x1855007617407872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077588.563942:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617407872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077588.563946:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077588.563948:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077588.563950:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617407872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077589.075298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077589.459345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077589.459354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077589.459419:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077590.099344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077590.483322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077590.483322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077590.483383:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077591.123347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077591.507355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077591.507370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077591.507453:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077592.147339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077592.531302:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077592.531315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077592.531362:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077593.171353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077593.555341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077593.555365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077593.555434:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077593.683345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077593.683354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077593.683357:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077593.683366:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4582 00000100:00080000:3.0:1769077593.683375:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077593.683383:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077593.683435:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617408000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077593.683464:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077593.683468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077593.683470:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077593.683475:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4582 00000100:00080000:3.0:1769077593.683479:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077593.683484:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077593.683494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077593.683510:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617408000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077593.683511:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617408128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077593.683529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077593.683533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077593.683535:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077593.683540:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4582 00000100:00080000:3.0:1769077593.683544:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077593.683549:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077593.683554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077593.683568:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617408256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077593.683578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077593.683582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077593.683584:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077593.683588:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4582 00000100:00080000:3.0:1769077593.683593:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077593.683598:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077593.683602:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077593.683610:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077593.683623:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617408384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077593.683633:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077593.683637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077593.683639:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077593.683645:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4582 00000100:00100000:0.0:1769077593.683649:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077593/real 1769077593] req@ffff9b0352a18000 x1855007617408000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077593.683651:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077593.683661:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077593.683670:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077593.683681:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617408000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077593.683687:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617408512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077593.683704:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077593.683713:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077593.683719:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617408000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077593.683777:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077593.683805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077593.683820:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077593/real 1769077593] req@ffff9b0352a18380 x1855007617408128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077593.683835:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617408128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077593.683844:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077593.683848:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077593.683852:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077593.683859:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077593.683880:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077593.683892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077593/real 1769077593] req@ffff9b0352a19c00 x1855007617408256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077593.683904:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617408256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077593.683913:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077593.683917:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077593.683921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077593.683927:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077593.683948:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077593.683960:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077593/real 1769077593] req@ffff9b0352a1b800 x1855007617408384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077593.683990:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617408384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077593.684000:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077593.684004:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077593.684008:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077593.684014:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077593.684036:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077593.684047:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077593/real 1769077593] req@ffff9b0352a19500 x1855007617408512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077593.684060:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617408512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077593.684069:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077593.684072:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077593.684076:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077594.195339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077594.579303:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077594.579318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077594.579386:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077595.219348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077595.603370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077595.603384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077595.603417:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077596.243347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077596.627351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077596.627359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077596.627420:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077597.267339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077597.651345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077597.651345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077597.651445:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077598.291345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077598.675344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077598.675344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077598.675413:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077598.803363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077598.803374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077598.803379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077598.803390:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4587 00000100:00080000:3.0:1769077598.803399:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077598.803407:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077598.803456:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617408640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077598.803483:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077598.803486:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077598.803488:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077598.803493:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4587 00000100:00080000:3.0:1769077598.803498:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077598.803504:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077598.803514:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077598.803523:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617408640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077598.803531:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617408768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077598.803545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077598.803549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077598.803552:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077598.803556:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4587 00000100:00080000:3.0:1769077598.803561:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077598.803565:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077598.803570:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769077598.803580:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077598.803585:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617408896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077598.803598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077598.803602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077598.803605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077598.803609:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4587 00000100:00080000:3.0:1769077598.803614:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077598.803616:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077598/real 1769077598] req@ffff9b0352a19500 x1855007617408640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077598.803622:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077598.803632:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077598.803639:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617408640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077598.803649:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617409024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077598.803678:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077598.803686:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769077598.803692:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077598.803693:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617408640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077598.803702:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077598.803705:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077598.803711:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4587 00000100:00080000:3.0:1769077598.803716:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077598.803720:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408768:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769077598.803725:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077598.803733:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077598.803750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617409152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077598.803752:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077598.803768:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077598/real 1769077598] req@ffff9b0352a1b800 x1855007617408768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077598.803786:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617408768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077598.803796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077598.803799:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077598.803803:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077598.803810:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077598.803831:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077598.803843:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077598/real 1769077598] req@ffff9b0352a19c00 x1855007617408896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077598.803853:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617408896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077598.803859:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077598.803861:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077598.803863:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617408896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077598.803871:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077598.803891:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077598.803902:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077598/real 1769077598] req@ffff9b0352a18380 x1855007617409024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077598.803915:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617409024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077598.803923:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077598.803927:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077598.803929:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077598.803941:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077598.803956:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077598.803963:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077598/real 1769077598] req@ffff9b0352a18000 x1855007617409152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077598.803970:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617409152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077598.803975:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077598.803977:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077598.803979:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077599.315348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077599.699334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077599.699334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077599.699391:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077600.339336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077600.723349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077600.723349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077600.723410:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077601.363336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077601.747320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077601.747378:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077601.747390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077602.387411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077602.771360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077602.771360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077602.771406:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077603.411365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077603.795364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077603.795371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077603.795430:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077603.923406:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077603.923415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077603.923418:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077603.923426:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4593 00000100:00080000:2.0:1769077603.923434:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077603.923441:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077603.923476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617409280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077603.923508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077603.923511:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077603.923513:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077603.923516:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4593 00000100:00080000:2.0:1769077603.923520:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077603.923524:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077603.923532:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077603.923544:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617409408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077603.923554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077603.923560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1769077603.923562:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617409280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077603.923563:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077603.923588:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4593 00000100:00080000:2.0:1769077603.923590:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077603.923596:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077603.923606:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077603.923620:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617409536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077603.923628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077603.923630:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077603.923631:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077603.923636:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4593 00000100:00080000:2.0:1769077603.923639:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077603.923646:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077603.923652:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077603.923668:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617409664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077603.923678:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077603.923681:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077603.923684:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077603.923696:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4593 00000800:00000100:3.0:1769077603.923699:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077603.923703:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077603.923710:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077603.923714:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077603.923725:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617409792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077603.923925:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077603/real 1769077603] req@ffff9b03482c5880 x1855007617409280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077603.923949:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617409280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077603.923961:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077603.923966:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077603.923971:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617409280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077603.924079:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077603.924106:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077603.924117:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077603/real 1769077603] req@ffff9b03482c5c00 x1855007617409408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077603.924127:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617409408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077603.924134:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077603.924137:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077603.924140:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077603.924145:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077603.924158:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077603.924173:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077603/real 1769077603] req@ffff9b03482c5f80 x1855007617409536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077603.924181:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617409536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077603.924187:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077603.924189:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077603.924191:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077603.924195:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077603.924207:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077603.924213:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077603/real 1769077603] req@ffff9b03482c7b80 x1855007617409664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077603.924222:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617409664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077603.924227:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077603.924229:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077603.924231:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077603.924263:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077603.924279:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077603.924290:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077603/real 1769077603] req@ffff9b03482c4e00 x1855007617409792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077603.924302:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617409792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077603.924307:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077603.924309:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077603.924311:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617409792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077604.435366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077604.819312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077604.819315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077604.819399:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077605.459352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077605.843355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077605.843355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077605.843427:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077606.483343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077606.867358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077606.867358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077606.867433:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077607.507355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077607.891350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077607.891350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077607.891416:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077608.531360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077608.915344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077608.915344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077608.915413:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077609.043352:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077609.043361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077609.043365:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077609.043373:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4598 00000100:00080000:2.0:1769077609.043383:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077609.043389:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077609.043423:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617409920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077609.043443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077609.043446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077609.043448:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077609.043452:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4598 00000100:00080000:2.0:1769077609.043455:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077609.043460:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077609.043471:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077609.043502:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617410048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077609.043502:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617409920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077609.043520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077609.043523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077609.043525:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077609.043528:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4598 00000100:00080000:2.0:1769077609.043531:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077609.043534:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077609.043538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077609.043549:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617410176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077609.043555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077609.043558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077609.043559:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077609.043562:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4598 00000100:00080000:2.0:1769077609.043566:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077609.043568:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077609.043572:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077609.043579:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077609.043580:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617410304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077609.043588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077609.043591:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077609.043592:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077609.043595:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4598 00000100:00080000:2.0:1769077609.043598:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077609.043600:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077609.043603:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077609.043610:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617410432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077609.043631:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff9b03482c4700 x1855007617409920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077609.043655:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617409920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077609.043666:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077609.043671:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077609.043676:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617409920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077609.043699:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077609.043725:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077609.043737:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff9b03482c6a00 x1855007617410048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077609.043751:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617410048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077609.043760:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077609.043764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077609.043767:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077609.043773:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077609.043794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077609.043806:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff9b03482c4000 x1855007617410176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077609.043820:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617410176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077609.043835:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077609.043839:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077609.043842:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077609.043849:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077609.043867:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077609.043877:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff9b03482c7100 x1855007617410304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077609.043890:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617410304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077609.043898:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077609.043901:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077609.043905:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077609.043911:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077609.043943:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077609.043955:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077609/real 1769077609] req@ffff9b03482c6680 x1855007617410432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077609.043968:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617410432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077609.043976:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077609.043979:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077609.043983:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077609.555359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077609.939310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077609.939310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077609.939381:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077610.579351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077610.963361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077610.963361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077610.963423:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077611.603359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077611.987320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077611.987323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077611.987364:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077612.627349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077613.011373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077613.011374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077613.011479:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077613.651341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077614.035299:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077614.035312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077614.035390:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077614.163343:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077614.163354:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077614.163359:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077614.163370:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4603 00000100:00080000:2.0:1769077614.163380:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077614.163388:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077614.163457:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617410560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077614.163489:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077614.163493:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077614.163495:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077614.163499:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4603 00000100:00080000:2.0:1769077614.163503:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077614.163508:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077614.163518:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077614.163531:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617410688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077614.163539:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617410560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077614.163548:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077614.163553:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077614.163555:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077614.163561:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4603 00000100:00080000:2.0:1769077614.163565:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077614.163569:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077614.163575:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077614.163589:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617410816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077614.163599:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077614.163600:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077614.163606:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077614.163609:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077614.163615:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4603 00000100:00080000:2.0:1769077614.163620:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077614.163624:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077614.163629:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077614.163645:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077614/real 1769077614] req@ffff9b03482c6d80 x1855007617410560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077614.163646:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617410944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077614.163681:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077614.163685:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077614.163688:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077614.163690:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617410560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077614.163695:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4603 00000100:00080000:2.0:1769077614.163704:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077614.163706:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077614.163711:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077614.163713:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617410560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077614.163722:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077614.163728:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077614.163739:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077614.163744:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617411072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077614.163764:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077614.163776:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077614/real 1769077614] req@ffff9b03482c6300 x1855007617410688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077614.163785:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617410688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077614.163791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077614.163793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077614.163796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077614.163800:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077614.163811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077614.163817:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077614/real 1769077614] req@ffff9b03482c5500 x1855007617410816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077614.163824:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617410816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077614.163829:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077614.163831:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077614.163833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077614.163837:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077614.163847:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077614.163852:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077614/real 1769077614] req@ffff9b03482c4a80 x1855007617410944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077614.163860:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617410944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077614.163865:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077614.163867:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077614.163868:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617410944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077614.163878:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077614.163887:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077614.163893:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077614/real 1769077614] req@ffff9b03482c5180 x1855007617411072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077614.163904:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617411072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077614.163912:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077614.163915:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077614.163919:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077614.675339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077615.059348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077615.059348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077615.059447:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077615.699321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077616.083303:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077616.083326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077616.083401:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077616.723375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077617.107400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077617.107407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077617.107466:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077617.747314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077618.131373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077618.131373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077618.131415:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077618.771356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077619.155352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077619.155358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077619.155437:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077619.283408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077619.283419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077619.283424:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077619.283435:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4608 00000100:00080000:1.0:1769077619.283444:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077619.283452:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077619.283488:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617411200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077619.283515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077619.283518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077619.283520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077619.283525:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4608 00000100:00080000:1.0:1769077619.283528:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077619.283532:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077619.283542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077619.283554:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617411328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077619.283565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077619.283569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077619.283572:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769077619.283574:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617411200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077619.283580:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4608 00000100:00080000:1.0:1769077619.283585:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077619.283590:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077619.283596:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077619.283609:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617411456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077619.283618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077619.283621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077619.283623:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077619.283627:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4608 00000100:00080000:1.0:1769077619.283631:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077619.283635:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077619.283639:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077619.283643:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077619.283653:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617411584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077619.283663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077619.283666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077619.283668:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077619.283672:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4608 00000100:00080000:1.0:1769077619.283676:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077619.283681:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077619.283686:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077619.283688:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077619/real 1769077619] req@ffff9b0343dedf80 x1855007617411200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077619.283699:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617411712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077619.283715:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617411200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077619.283729:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077619.283734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077619.283738:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617411200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077619.283761:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077619.283781:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077619.283800:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077619/real 1769077619] req@ffff9b0343def100 x1855007617411328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077619.283809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617411328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077619.283815:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077619.283818:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077619.283820:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077619.283824:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077619.283836:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077619.283842:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077619/real 1769077619] req@ffff9b0343dee300 x1855007617411456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077619.283850:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617411456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077619.283855:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077619.283857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077619.283859:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077619.283863:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077619.283873:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077619.283878:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077619/real 1769077619] req@ffff9b0343dec380 x1855007617411584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077619.283885:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617411584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077619.283891:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077619.283893:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077619.283895:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077619.283898:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077619.283907:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077619.283912:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077619/real 1769077619] req@ffff9b0343ded180 x1855007617411712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077619.283918:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617411712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077619.283923:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077619.283924:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077619.283926:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077619.795350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077620.179350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077620.179351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077620.179382:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077620.819348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077621.203337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077621.203338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077621.203402:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077621.843339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077622.227370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077622.227371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077622.227393:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077622.867384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077623.251384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077623.251386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077623.251452:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077623.891375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077624.275404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077624.275409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077624.275446:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077624.403395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077624.403408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077624.403413:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077624.403424:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4613 00000100:00080000:3.0:1769077624.403435:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077624.403442:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077624.403492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617411840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077624.403528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077624.403532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077624.403539:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077624.403543:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4613 00000100:00080000:3.0:1769077624.403547:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077624.403551:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077624.403561:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077624.403574:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617411968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077624.403580:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617411840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077624.403585:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077624.403589:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077624.403591:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077624.403594:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4613 00000100:00080000:3.0:1769077624.403596:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077624.403599:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077624.403603:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077624.403612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617412096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077624.403619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077624.403621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077624.403623:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077624.403627:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4613 00000100:00080000:3.0:1769077624.403630:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077624.403633:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077624.403636:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077624.403643:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617412224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077624.403666:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077624.403669:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077624.403670:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077624.403672:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4613 00000100:00080000:3.0:1769077624.403675:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077624.403680:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769077624.403684:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769077624.403686:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077624.403697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617412352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077624.403748:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077624/real 1769077624] req@ffff9b0352a1ad80 x1855007617411840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077624.403773:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617411840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077624.403786:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077624.403792:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077624.403797:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617411840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077624.403845:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077624.403872:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077624.403886:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077624/real 1769077624] req@ffff9b0352a1b480 x1855007617411968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077624.403901:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617411968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077624.403912:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077624.403915:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077624.403919:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617411968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077624.403926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077624.403946:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077624.403958:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077624/real 1769077624] req@ffff9b0352a18a80 x1855007617412096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077624.403970:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617412096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077624.403978:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077624.403982:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077624.403985:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077624.404022:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077624.404047:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077624.404074:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077624/real 1769077624] req@ffff9b0352a1a680 x1855007617412224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077624.404087:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617412224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077624.404102:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077624.404106:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077624.404109:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077624.404116:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077624.404163:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077624.404183:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077624/real 1769077624] req@ffff9b0352a1b100 x1855007617412352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077624.404218:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617412352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077624.404302:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077624.404305:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077624.404307:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077624.915391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077625.299399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077625.299738:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077625.299938:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077625.939322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077626.323369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077626.323648:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077626.324226:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077626.963344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077627.347345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077627.347346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077627.347491:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077627.987297:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077628.371484:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077628.371841:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077628.371868:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077629.011342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077629.395338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077629.395670:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077629.396124:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077629.523389:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077629.523398:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077629.523401:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077629.523409:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4618 00000100:00080000:0.0:1769077629.523417:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077629.523423:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077629.523456:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617412480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077629.523489:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077629.523493:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077629.523495:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077629.523500:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4618 00000100:00080000:0.0:1769077629.523504:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077629.523509:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077629.523518:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077629.523538:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617412608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077629.523541:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617412480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077629.523551:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077629.523555:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077629.523556:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077629.523559:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4618 00000100:00080000:0.0:1769077629.523562:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077629.523565:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077629.523569:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077629.523584:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617412736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077629.523590:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077629.523593:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077629.523594:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077629.523596:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4618 00000100:00080000:0.0:1769077629.523599:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077629.523602:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077629.523606:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077629.523613:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617412864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077629.523621:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077629.523623:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077629.523627:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077629.523629:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077629.523632:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4618 00000100:00080000:0.0:1769077629.523635:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077629.523638:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077629.523641:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077629.523647:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617412992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077629.523673:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077629/real 1769077629] req@ffff9b0365a3b100 x1855007617412480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077629.523695:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617412480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077629.523705:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077629.523709:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077629.523714:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617412480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077629.523737:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077629.523762:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077629.523774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077629/real 1769077629] req@ffff9b0365a39500 x1855007617412608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077629.523787:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617412608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077629.523795:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077629.523798:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077629.523802:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077629.523808:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077629.523827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077629.523837:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077629/real 1769077629] req@ffff9b0365a3bb80 x1855007617412736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077629.523856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617412736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077629.523865:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077629.523868:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077629.523871:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077629.523877:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077629.523896:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077629.523907:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077629/real 1769077629] req@ffff9b0365a3ad80 x1855007617412864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077629.523920:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617412864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077629.523928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077629.523932:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077629.523935:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077629.523951:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077629.523971:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077629.523981:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077629/real 1769077629] req@ffff9b0365a39180 x1855007617412992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077629.523992:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617412992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077629.524002:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077629.524004:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077629.524006:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617412992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077630.035336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077630.419387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077630.419434:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077630.419657:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077631.059342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077631.443316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077631.443367:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077631.443411:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077632.083350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077632.467362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077632.467400:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077632.467502:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077633.107337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077633.491303:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077633.491308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077633.491357:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077634.131336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077634.515364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077634.515369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077634.515411:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077634.643324:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077634.643333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077634.643336:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077634.643344:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4623 00000100:00080000:3.0:1769077634.643352:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077634.643358:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077634.643396:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617413120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077634.643427:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077634.643431:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077634.643433:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077634.643437:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4623 00000100:00080000:3.0:1769077634.643441:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077634.643445:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077634.643453:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077634.643465:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617413120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077634.643466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617413248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077634.643480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077634.643482:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077634.643484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077634.643487:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4623 00000100:00080000:3.0:1769077634.643490:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077634.643492:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077634.643495:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077634.643503:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617413376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077634.643509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077634.643512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077634.643513:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077634.643515:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4623 00000100:00080000:3.0:1769077634.643518:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077634.643520:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077634.643523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769077634.643526:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077634.643531:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617413504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077634.643539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077634.643542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077634.643544:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077634.643546:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4623 00000100:00080000:3.0:1769077634.643549:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077634.643551:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077634.643554:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077634.643566:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077634/real 1769077634] req@ffff9b0352a1aa00 x1855007617413120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077634.643566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617413632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077634.643597:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617413120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077634.643605:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077634.643609:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077634.643612:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617413120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077634.643629:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617413248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077634.643644:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077634.643651:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077634/real 1769077634] req@ffff9b0352a18700 x1855007617413248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077634.643660:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617413248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077634.643665:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077634.643668:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077634.643670:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617413248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077634.643674:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617413376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077634.643684:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077634.643689:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077634/real 1769077634] req@ffff9b0352a19f80 x1855007617413376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077634.643697:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617413376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077634.643702:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077634.643703:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077634.643705:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617413376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077634.643709:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617413504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077634.643718:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077634.643724:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077634/real 1769077634] req@ffff9b0352a1a300 x1855007617413504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077634.643732:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617413504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077634.643737:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077634.643739:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077634.643741:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617413504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077634.643744:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617413632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077634.643753:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077634.643758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077634/real 1769077634] req@ffff9b0352a18e00 x1855007617413632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077634.643765:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617413632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077634.643770:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077634.643772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077634.643774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617413632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077635.155339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077635.539334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077635.539335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077635.539428:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077636.179346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077636.563336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077636.563339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077636.563411:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077637.203320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077637.587340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077637.587341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077637.587433:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077638.227374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077638.611407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077638.611408:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077638.611459:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077639.251316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077639.635350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077639.635350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077639.635475:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077639.763356:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077639.763366:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077639.763370:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077639.763378:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4628 00000100:00080000:3.0:1769077639.763388:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077639.763394:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077639.763428:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617413760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077639.763458:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077639.763464:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077639.763466:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077639.763469:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4628 00000100:00080000:3.0:1769077639.763475:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077639.763479:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077639.763489:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077639.763501:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617413888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077639.763510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077639.763513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769077639.763514:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617413760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077639.763515:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077639.763523:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4628 00000100:00080000:3.0:1769077639.763526:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077639.763529:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077639.763531:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077639.763542:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617414016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077639.763547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077639.763550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077639.763552:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077639.763555:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4628 00000100:00080000:3.0:1769077639.763558:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077639.763561:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077639.763565:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077639.763580:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617414144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077639.763593:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077639.763595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077639.763600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077639.763603:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077639.763609:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4628 00000100:00080000:3.0:1769077639.763616:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077639.763622:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077639.763628:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077639.763643:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077639/real 1769077639] req@ffff9b0352a18e00 x1855007617413760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077639.763647:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617414272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077639.763672:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617413760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077639.763681:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077639.763685:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077639.763689:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617413760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077639.763715:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617413888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077639.763748:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077639.763757:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077639/real 1769077639] req@ffff9b0352a1a300 x1855007617413888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077639.763769:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617413888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077639.763778:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077639.763780:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077639.763783:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617413888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077639.763788:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077639.763801:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077639.763815:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077639/real 1769077639] req@ffff9b0352a19f80 x1855007617414016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077639.763831:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617414016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077639.763840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077639.763844:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077639.763847:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077639.763854:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077639.763877:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077639.763889:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077639/real 1769077639] req@ffff9b0352a18700 x1855007617414144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077639.763898:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617414144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077639.763904:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077639.763907:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077639.763909:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077639.763914:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077639.763930:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077639.763939:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077639/real 1769077639] req@ffff9b0352a1aa00 x1855007617414272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077639.763949:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617414272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077639.763955:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077639.763957:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077639.763959:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077640.275406:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077640.659374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077640.659374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077640.659442:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077641.299354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077641.683390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077641.683393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077641.683469:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077642.323359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077642.707322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077642.707322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077642.707404:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077643.347298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077643.731345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077643.731353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077643.731385:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077644.371340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077644.755293:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077644.755295:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077644.755383:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077644.883347:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077644.883356:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077644.883360:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077644.883369:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4634 00000100:00080000:3.0:1769077644.883377:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077644.883383:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077644.883417:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617414400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077644.883439:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077644.883442:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077644.883443:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077644.883447:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4634 00000100:00080000:3.0:1769077644.883450:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077644.883453:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077644.883460:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077644.883474:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617414400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077644.883476:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617414528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077644.883491:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077644.883495:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077644.883498:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077644.883512:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4634 00000100:00080000:3.0:1769077644.883530:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077644.883534:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077644.883539:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077644.883549:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617414656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077644.883557:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077644.883559:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077644.883561:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077644.883564:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4634 00000100:00080000:3.0:1769077644.883567:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077644.883570:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077644.883574:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077644.883582:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617414784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077644.883585:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414528:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769077644.883601:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077644.883603:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077644.883605:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077644.883610:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4634 00000100:00080000:3.0:1769077644.883615:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077644.883620:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077644.883621:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414656:192.168.204.160@tcp:38:kworker.0 00000008:00100000:3.0:1769077644.883629:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077644.883647:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617414912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077644.883651:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077644.883681:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077645.395298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077645.779343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077645.779343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077645.779400:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077646.419340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077646.803330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077646.803351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077646.803411:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077647.443286:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077647.827356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077647.827356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077647.827417:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769077647.955492:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769077647.955536:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769077647.955553:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077647.955593:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077647.955603:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077647.955615:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077647.955627:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769077647.955640:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077644/real 1769077647] req@ffff9b0352a1b480 x1855007617414784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077647.955667:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617414784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077647.955679:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077647.955683:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077647.955688:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077647.955695:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077644/real 1769077647] req@ffff9b0352a1a680 x1855007617414528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077647.955707:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617414528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077647.955714:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077647.955717:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077647.955721:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077647.955728:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077644/real 1769077647] req@ffff9b0352a18a80 x1855007617414656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077647.955744:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617414656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077647.955751:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077647.955754:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077647.955757:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077647.955763:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077644/real 1769077647] req@ffff9b0352a1b100 x1855007617414400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077647.955774:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617414400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077647.955781:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077647.955784:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077647.955787:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617414400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077647.955814:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077644/real 1769077647] req@ffff9b0352a1ad80 x1855007617414912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077647.955825:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617414912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077647.955846:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077647.955849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077647.955853:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617414912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077648.467280:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077648.851320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077648.851431:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077648.851470:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077649.491336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077649.875382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077649.875389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077649.875502:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077650.003318:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077650.003327:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077650.003331:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077650.003339:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4639 00000100:00080000:3.0:1769077650.003347:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077650.003352:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077650.003382:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617415040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077650.003404:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077650.003407:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077650.003408:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077650.003411:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4639 00000100:00080000:3.0:1769077650.003414:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077650.003417:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077650.003425:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077650.003436:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617415168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077650.003437:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617415040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077650.003451:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077650.003454:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077650.003455:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077650.003458:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4639 00000100:00080000:3.0:1769077650.003460:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077650.003462:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077650.003466:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077650.003473:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617415296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077650.003480:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077650.003483:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077650.003484:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077650.003486:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4639 00000100:00080000:3.0:1769077650.003495:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077650.003497:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077650.003500:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077650.003509:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617415424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077650.003514:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077650.003521:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077650.003524:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077650.003526:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077650.003529:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4639 00000100:00080000:3.0:1769077650.003532:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077650.003535:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077650.003538:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077650.003546:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617415552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077650.003567:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077650] req@ffff9b0352a18000 x1855007617415040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077650.003586:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617415040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077650.003595:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077650.003598:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077650.003601:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617415040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077650.003620:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077650.003640:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077650.003649:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077650] req@ffff9b0352a18380 x1855007617415168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077650.003658:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617415168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077650.003663:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077650.003666:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077650.003668:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077650.003672:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077650.003683:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077650.003696:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077650] req@ffff9b0352a19c00 x1855007617415296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077650.003711:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617415296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077650.003717:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077650.003719:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077650.003722:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077650.003729:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077650.003748:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077650.003760:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077650] req@ffff9b0352a1b800 x1855007617415424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077650.003772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617415424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077650.003782:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077650.003786:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077650.003791:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077650.003818:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077650.003839:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077650.003849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077650/real 1769077650] req@ffff9b0352a19500 x1855007617415552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077650.003861:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617415552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077650.003875:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077650.003878:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077650.003882:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077650.515291:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077650.899316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077650.899346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077650.899403:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077651.539301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077651.923341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077651.923341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077651.923396:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077652.563395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077652.947305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077652.947319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077652.947410:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077653.587313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077653.971347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077653.971347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077653.971431:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077654.611312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077654.995314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077654.995351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077654.995399:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077655.123341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077655.123351:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077655.123355:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077655.123363:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4644 00000100:00080000:3.0:1769077655.123371:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077655.123377:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077655.123409:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617415680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077655.123432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077655.123435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077655.123436:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077655.123439:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4644 00000100:00080000:3.0:1769077655.123442:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077655.123444:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077655.123451:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077655.123460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617415808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077655.123467:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617415680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077655.123477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077655.123482:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077655.123484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077655.123499:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4644 00000100:00080000:3.0:1769077655.123505:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077655.123510:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077655.123515:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077655.123528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617415936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077655.123536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077655.123538:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077655.123539:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769077655.123545:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077655.123546:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4644 00000100:00080000:3.0:1769077655.123552:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077655.123556:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077655.123560:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077655.123569:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617416064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077655.123576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077655.123580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077655.123582:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077655.123586:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4644 00000100:00100000:2.0:1769077655.123589:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077655/real 1769077655] req@ffff9b0352a19500 x1855007617415680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077655.123592:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077655.123602:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077655.123608:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077655.123610:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617415680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077655.123620:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077655.123625:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077655.123626:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617416192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077655.123629:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617415680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077655.123654:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077655.123673:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077655.123681:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077655/real 1769077655] req@ffff9b0352a1b800 x1855007617415808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077655.123689:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617415808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077655.123699:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077655.123701:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077655.123704:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077655.123724:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077655.123739:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077655.123747:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077655/real 1769077655] req@ffff9b0352a19c00 x1855007617415936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077655.123754:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617415936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077655.123760:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077655.123762:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077655.123766:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617415936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077655.123772:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077655.123791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077655.123803:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077655/real 1769077655] req@ffff9b0352a18380 x1855007617416064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077655.123816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617416064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077655.123824:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077655.123828:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077655.123832:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077655.123836:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077655.123851:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077655.123858:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077655/real 1769077655] req@ffff9b0352a18000 x1855007617416192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077655.123866:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617416192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077655.123871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077655.123873:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077655.123875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077655.635360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077656.019332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077656.019385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077656.019405:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077656.659431:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077657.043345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077657.043345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077657.043413:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077657.683333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077658.067321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077658.067323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077658.067380:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077658.707339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077659.091340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077659.091340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077659.091414:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077659.731335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077660.115366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077660.115413:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077660.115442:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:0.0:1769077660.243412:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077660.243423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077660.243427:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077660.243436:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4649 00000100:00080000:0.0:1769077660.243445:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077660.243452:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077660.243493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617416320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077660.243523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077660.243527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077660.243529:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077660.243534:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4649 00000100:00080000:0.0:1769077660.243538:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077660.243543:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077660.243552:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077660.243562:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617416320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077660.243568:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617416448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077660.243578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077660.243581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077660.243582:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077660.243585:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4649 00000100:00080000:0.0:1769077660.243588:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077660.243592:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077660.243595:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077660.243603:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617416576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077660.243610:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077660.243612:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077660.243614:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077660.243616:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4649 00000100:00080000:0.0:1769077660.243618:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077660.243637:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077660.243641:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769077660.243647:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077660.243651:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617416704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077660.243658:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077660.243661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077660.243662:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077660.243665:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4649 00000100:00080000:0.0:1769077660.243668:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077660.243670:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077660.243673:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077660.243682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617416832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077660.243689:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077660/real 1769077660] req@ffff9b0365a39180 x1855007617416320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077660.243708:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617416320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077660.243720:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077660.243725:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077660.243730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617416320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077660.243755:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077660.243782:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077660.243795:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077660/real 1769077660] req@ffff9b0365a3ad80 x1855007617416448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077660.243809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617416448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077660.243818:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077660.243822:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077660.243825:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077660.243832:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077660.243852:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077660.243864:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077660/real 1769077660] req@ffff9b0365a3bb80 x1855007617416576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077660.243877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617416576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077660.243886:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077660.243890:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077660.243894:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077660.243900:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077660.243918:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077660.243930:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077660/real 1769077660] req@ffff9b0365a39500 x1855007617416704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077660.243942:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617416704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077660.243951:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077660.243954:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077660.243958:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077660.243964:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077660.243983:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077660.243993:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077660/real 1769077660] req@ffff9b0365a3b100 x1855007617416832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077660.244006:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617416832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077660.244015:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077660.244018:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077660.244022:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617416832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077660.755301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077661.139342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077661.139342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077661.139401:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077661.779353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077662.163337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077662.163362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077662.163414:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077662.803285:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077663.187369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077663.187401:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077663.187405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077663.827367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077664.211292:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077664.211359:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077664.211386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077664.851306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077665.235348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077665.235348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077665.235449:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077665.363364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077665.363376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077665.363381:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077665.363391:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4654 00000100:00080000:3.0:1769077665.363402:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077665.363409:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077665.363452:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617416960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077665.363480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077665.363484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077665.363486:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077665.363489:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4654 00000100:00080000:3.0:1769077665.363494:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077665.363497:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077665.363508:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077665.363519:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617417088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077665.363530:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617416960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077665.363532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077665.363549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077665.363551:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077665.363555:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4654 00000100:00080000:3.0:1769077665.363558:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077665.363564:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077665.363582:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077665.363598:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617417216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077665.363605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077665.363608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077665.363610:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077665.363613:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4654 00000100:00080000:3.0:1769077665.363627:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077665.363631:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077665.363638:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769077665.363645:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077665.363651:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617417344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077665.363661:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077665.363664:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077665.363666:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077665.363670:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4654 00000100:00080000:3.0:1769077665.363672:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077665.363676:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077665.363679:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077665.363688:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617417472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077665.363708:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077665/real 1769077665] req@ffff9b0352a1ad80 x1855007617416960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077665.363730:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617416960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077665.363739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077665.363743:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077665.363747:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617416960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077665.363774:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077665.363803:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077665.363815:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077665/real 1769077665] req@ffff9b0352a1b100 x1855007617417088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077665.363824:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617417088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077665.363830:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077665.363833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077665.363836:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077665.363843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077665.363864:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077665.363876:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077665/real 1769077665] req@ffff9b0352a18a80 x1855007617417216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077665.363889:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617417216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077665.363902:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077665.363905:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077665.363907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077665.363912:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077665.363928:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077665.363935:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077665/real 1769077665] req@ffff9b0352a1a680 x1855007617417344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077665.363942:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617417344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077665.363947:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077665.363949:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077665.363952:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077665.363955:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077665.363965:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077665.363970:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077665/real 1769077665] req@ffff9b0352a1b480 x1855007617417472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077665.363976:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617417472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077665.363981:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077665.363983:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077665.363986:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077665.875317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077666.259327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077666.259327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077666.259389:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077666.899286:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077667.283346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077667.283346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077667.283390:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077667.923362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077668.307334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077668.307335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077668.307366:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077668.947350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077669.331370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077669.331370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077669.331462:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077669.971337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077670.355307:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077670.355310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077670.355360:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077670.483375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077670.483388:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077670.483392:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077670.483402:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4659 00000100:00080000:1.0:1769077670.483413:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077670.483420:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077670.483497:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617417600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077670.483526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077670.483530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077670.483532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077670.483537:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4659 00000100:00080000:1.0:1769077670.483542:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077670.483547:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077670.483558:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077670.483558:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617417600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077670.483579:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617417728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077670.483591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077670.483596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077670.483598:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077670.483603:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4659 00000100:00080000:1.0:1769077670.483607:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077670.483612:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077670.483618:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077670.483623:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077670.483635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617417856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077670.483647:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077670.483650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077670.483652:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077670.483659:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4659 00000100:00100000:2.0:1769077670.483659:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077670/real 1769077670] req@ffff9b0343dec700 x1855007617417600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077670.483667:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077670.483675:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077670.483681:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077670.483690:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617417600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077670.483695:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617417984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077670.483701:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077670.483709:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077670.483715:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617417600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077670.483716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077670.483724:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077670.483726:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077670.483732:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4659 00000100:00080000:1.0:1769077670.483737:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769077670.483740:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417728:192.168.204.160@tcp:38:kworker.0 00000100:00080000:1.0:1769077670.483745:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077670.483752:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769077670.483763:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077670.483767:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617418112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077670.483774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077670/real 1769077670] req@ffff9b0343dec000 x1855007617417728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077670.483793:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617417728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077670.483800:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077670.483802:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077670.483805:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077670.483811:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077670.483821:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077670.483827:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077670/real 1769077670] req@ffff9b0343ded180 x1855007617417856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077670.483834:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617417856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077670.483839:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077670.483841:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077670.483843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077670.483847:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077670.483856:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077670.483862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077670/real 1769077670] req@ffff9b0343dec380 x1855007617417984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077670.483869:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617417984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077670.483874:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077670.483875:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077670.483877:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617417984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077670.483886:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617418112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077670.483895:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077670.483901:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077670/real 1769077670] req@ffff9b0343dee300 x1855007617418112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077670.483908:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617418112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077670.483913:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077670.483915:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077670.483917:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617418112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077670.995339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077671.379349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077671.379349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077671.379466:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077672.019343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077672.403290:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077672.403307:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077672.403383:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077673.043353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077673.427335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077673.427335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077673.427370:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077674.067336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077674.451366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077674.451367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077674.451460:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077675.091412:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077675.475355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077675.475355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077675.475456:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077675.603346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077675.603354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077675.603359:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077675.603389:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4664 00000100:00080000:1.0:1769077675.603398:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077675.603404:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077675.603439:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617418240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077675.603470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077675.603476:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077675.603477:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077675.603480:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4664 00000100:00080000:1.0:1769077675.603484:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077675.603487:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077675.603496:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077675.603507:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617418368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077675.603516:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617418240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077675.603518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077675.603525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077675.603527:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077675.603531:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4664 00000100:00080000:1.0:1769077675.603534:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077675.603537:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077675.603540:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077675.603551:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617418496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077675.603557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077675.603560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077675.603562:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077675.603565:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4664 00000100:00080000:1.0:1769077675.603569:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077675.603571:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077675.603575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077675.603584:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617418624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077675.603590:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077675.603593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077675.603594:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769077675.603600:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077675.603601:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4664 00000100:00080000:1.0:1769077675.603606:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077675.603609:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077675.603613:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077675.603622:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617418752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077675.603646:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077675/real 1769077675] req@ffff9b0343def100 x1855007617418240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077675.603679:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617418240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077675.603770:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077675.603775:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077675.603779:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617418240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077675.603801:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617418368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077675.603822:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077675.603832:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077675/real 1769077675] req@ffff9b0343dedf80 x1855007617418368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077675.603849:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617418368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077675.603855:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077675.603858:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077675.603860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617418368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077675.603864:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617418496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077675.603976:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077675.603988:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077675/real 1769077675] req@ffff9b0343ded880 x1855007617418496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077675.603998:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617418496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077675.604005:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077675.604008:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077675.604011:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617418496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077675.604016:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617418624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077675.604030:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077675.604042:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077675/real 1769077675] req@ffff9b0343def800 x1855007617418624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077675.604054:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617418624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077675.604060:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077675.604063:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077675.604065:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617418624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077675.604069:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617418752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077675.604080:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077675.604138:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077675/real 1769077675] req@ffff9b0343deea00 x1855007617418752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077675.604155:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617418752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077675.604162:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077675.604165:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077675.604168:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617418752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077676.115306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077676.499337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077676.499346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077676.499393:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077677.139462:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077677.523414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077677.523450:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077677.523553:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077678.163374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077678.547356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077678.547357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077678.547403:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077679.187372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077679.571317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077679.571339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077679.571386:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077680.211310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077680.595421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077680.595422:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077680.595514:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077680.723365:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077680.723377:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077680.723383:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077680.723394:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4669 00000100:00080000:3.0:1769077680.723405:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077680.723412:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077680.723455:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617418880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077680.723485:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077680.723488:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077680.723489:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077680.723493:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4669 00000100:00080000:3.0:1769077680.723497:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077680.723501:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077680.723510:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077680.723519:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617419008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077680.723532:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617418880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077680.723533:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077680.723548:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077680.723550:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077680.723553:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4669 00000100:00080000:3.0:1769077680.723555:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077680.723557:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077680.723561:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077680.723571:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617419136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077680.723577:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077680.723580:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077680.723581:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077680.723584:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4669 00000100:00080000:3.0:1769077680.723597:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077680.723602:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077680.723606:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769077680.723612:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077680.723618:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617419264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077680.723636:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077680.723640:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077680.723643:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077680.723649:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4669 00000100:00100000:2.0:1769077680.723657:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077680/real 1769077680] req@ffff9b0352a1aa00 x1855007617418880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077680.723658:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077680.723671:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077680.723686:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077680.723697:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617418880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077680.723705:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617419392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077680.723761:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077680.723767:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077680.723773:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617418880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077680.723800:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077680.723821:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077680.723830:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077680/real 1769077680] req@ffff9b0352a18700 x1855007617419008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077680.723840:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617419008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077680.723846:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077680.723848:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077680.723851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077680.723855:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077680.723865:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077680.723871:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077680/real 1769077680] req@ffff9b0352a19f80 x1855007617419136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077680.723878:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617419136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077680.723883:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077680.723885:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077680.723887:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077680.723891:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077680.723902:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077680.723907:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077680/real 1769077680] req@ffff9b0352a1a300 x1855007617419264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077680.723914:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617419264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077680.723920:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077680.723922:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077680.723924:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077680.723928:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077680.723937:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077680.723942:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077680/real 1769077680] req@ffff9b0352a18e00 x1855007617419392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077680.723960:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617419392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077680.723969:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077680.723972:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077680.723976:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077681.235371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077681.619343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077681.619365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077681.619399:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077682.259306:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077682.643367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077682.643368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077682.643431:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077683.283403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077683.667366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077683.667373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077683.667441:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077684.307288:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077684.691373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077684.691398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077684.691451:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077685.331403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077685.715382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077685.715383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077685.715463:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077685.843300:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077685.843312:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077685.843318:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077685.843329:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4675 00000100:00080000:3.0:1769077685.843340:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077685.843348:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077685.843403:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617419520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077685.843434:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077685.843437:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077685.843439:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077685.843442:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4675 00000100:00080000:3.0:1769077685.843445:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077685.843449:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077685.843460:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077685.843472:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617419648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077685.843482:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077685.843487:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077685.843489:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077685.843491:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617419520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077685.843495:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4675 00000100:00080000:3.0:1769077685.843500:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077685.843510:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077685.843515:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077685.843521:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617419776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077685.843528:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077685.843531:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077685.843532:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077685.843537:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4675 00000100:00080000:3.0:1769077685.843540:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077685.843542:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077685.843548:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077685.843556:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617419904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077685.843562:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077685.843564:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077685.843566:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077685.843570:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4675 00000100:00080000:3.0:1769077685.843573:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077685.843575:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077685.843578:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769077685.843585:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077685.843587:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617420032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077685.843639:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077685/real 1769077685] req@ffff9b0352a18e00 x1855007617419520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077685.843660:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617419520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077685.843673:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077685.843677:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077685.843682:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617419520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077685.843706:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077685.843728:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077685.843740:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077685/real 1769077685] req@ffff9b0352a1a300 x1855007617419648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077685.843754:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617419648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077685.843764:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077685.843768:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077685.843771:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077685.843778:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077685.843794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077685.843803:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077685/real 1769077685] req@ffff9b0352a19f80 x1855007617419776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077685.843822:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617419776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077685.843831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077685.843834:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077685.843838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077685.843845:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077685.843862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077685.843870:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077685/real 1769077685] req@ffff9b0352a18700 x1855007617419904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077685.843883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617419904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077685.843891:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077685.843894:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077685.843898:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617419904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077685.843904:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077685.843919:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077685.843928:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077685/real 1769077685] req@ffff9b0352a1aa00 x1855007617420032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077685.843940:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617420032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077685.843955:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077685.843959:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077685.843962:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077686.355377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077686.739320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077686.739359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077686.739359:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077687.379342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077687.763308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077687.763308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077687.763359:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077688.403338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077688.787333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077688.787333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077688.787424:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077689.427348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077689.811313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077689.811313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077689.811370:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077690.451359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077690.835356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077690.835357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077690.835420:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077690.963315:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077690.963324:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077690.963328:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077690.963337:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4680 00000100:00080000:3.0:1769077690.963344:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077690.963350:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077690.963384:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617420160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077690.963405:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077690.963408:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077690.963409:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077690.963413:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4680 00000100:00080000:3.0:1769077690.963415:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077690.963417:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077690.963431:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077690.963442:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617420288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077690.963449:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617420160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077690.963452:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077690.963459:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077690.963461:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077690.963464:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4680 00000100:00080000:3.0:1769077690.963466:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077690.963469:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077690.963472:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077690.963480:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617420416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077690.963486:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077690.963489:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077690.963490:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077690.963493:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4680 00000100:00080000:3.0:1769077690.963495:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077690.963497:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077690.963501:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077690.963508:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617420544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077690.963513:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077690.963516:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077690.963517:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077690.963520:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4680 00000100:00080000:3.0:1769077690.963522:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077690.963524:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077690.963527:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769077690.963534:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077690.963536:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617420672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077690.963583:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077690/real 1769077690] req@ffff9b0352a1b480 x1855007617420160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077690.963604:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617420160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077690.963616:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077690.963621:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077690.963626:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617420160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077690.963653:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077690.963679:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077690.963693:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077690/real 1769077690] req@ffff9b0352a1a680 x1855007617420288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077690.963708:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617420288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077690.963719:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077690.963723:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077690.963727:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077690.963734:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077690.963754:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077690.963766:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077690/real 1769077690] req@ffff9b0352a18a80 x1855007617420416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077690.963779:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617420416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077690.963789:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077690.963793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077690.963796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077690.963803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077690.963824:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077690.963836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077690/real 1769077690] req@ffff9b0352a1b100 x1855007617420544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077690.963850:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617420544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077690.963859:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077690.963862:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077690.963866:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077690.963873:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077690.963894:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077690.963905:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077690/real 1769077690] req@ffff9b0352a1ad80 x1855007617420672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077690.963918:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617420672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077690.963928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077690.963931:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077690.963935:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077691.475343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077691.859330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077691.859524:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077691.859680:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077692.499341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077692.883362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077692.883374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077692.883927:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077693.523360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077693.907345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077693.907514:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077693.907534:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077694.547345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077694.931527:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077694.931773:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077694.932056:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077695.571352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077695.955388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077695.955494:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077695.955886:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077696.083382:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077696.083393:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077696.083398:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077696.083409:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4685 00000100:00080000:0.0:1769077696.083418:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077696.083426:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077696.083464:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617420800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077696.083491:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077696.083495:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077696.083496:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077696.083499:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4685 00000100:00080000:0.0:1769077696.083503:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077696.083507:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077696.083514:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077696.083522:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617420800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077696.083528:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617420928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077696.083537:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077696.083540:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077696.083541:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077696.083544:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4685 00000100:00080000:0.0:1769077696.083547:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077696.083550:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077696.083554:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077696.083562:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617421056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077696.083568:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077696.083570:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077696.083572:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077696.083575:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4685 00000100:00080000:0.0:1769077696.083578:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077696.083580:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077696.083583:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769077696.083587:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077696.083593:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617421184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077696.083607:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077696.083610:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077696.083612:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077696.083615:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4685 00000100:00080000:0.0:1769077696.083618:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077696.083620:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077696.083623:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077696.083630:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617421312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077696.083642:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077696/real 1769077696] req@ffff9b0365a3aa00 x1855007617420800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077696.083666:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617420800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077696.083675:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077696.083678:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077696.083681:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617420800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077696.083700:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077696.083718:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077696.083727:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077696/real 1769077696] req@ffff9b0365a3b480 x1855007617420928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077696.083735:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617420928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077696.083741:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077696.083743:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077696.083745:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617420928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077696.083749:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077696.083759:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077696.083764:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077696/real 1769077696] req@ffff9b0365a38e00 x1855007617421056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077696.083772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617421056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077696.083777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077696.083778:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077696.083780:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077696.083784:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077696.083794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077696.083800:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077696/real 1769077696] req@ffff9b0365a39c00 x1855007617421184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077696.083807:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617421184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077696.083811:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077696.083813:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077696.083815:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077696.083818:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077696.083827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077696.083832:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077696/real 1769077696] req@ffff9b0365a38a80 x1855007617421312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077696.083839:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617421312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077696.083843:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077696.083845:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077696.083847:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077696.595340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077696.979338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077696.979744:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077696.980143:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077697.619332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077698.003340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077698.003340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077698.003418:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077698.643330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077699.027322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077699.027324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077699.027381:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077699.667304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077700.051347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077700.051347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077700.051418:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077700.691334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077701.075340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077701.075342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077701.075410:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077701.203343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077701.203351:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077701.203355:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077701.203368:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4690 00000100:00080000:1.0:1769077701.203377:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077701.203383:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077701.203420:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617421440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077701.203440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077701.203443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077701.203444:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077701.203447:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4690 00000100:00080000:1.0:1769077701.203450:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077701.203453:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077701.203461:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077701.203472:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617421568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077701.203475:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617421440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077701.203487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077701.203490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077701.203491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077701.203494:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4690 00000100:00080000:1.0:1769077701.203496:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077701.203498:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077701.203501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077701.203510:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617421696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077701.203517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077701.203519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077701.203520:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077701.203523:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4690 00000100:00080000:1.0:1769077701.203530:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077701.203534:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769077701.203539:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769077701.203541:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077701.203564:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617421824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077701.203576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077701.203582:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769077701.203582:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077701/real 1769077701] req@ffff9b0343deca80 x1855007617421440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077701.203585:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077701.203595:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4690 00000100:00080000:1.0:1769077701.203601:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077701.203605:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077701.203610:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077701.203616:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617421440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077701.203621:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617421952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077701.203642:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077701.203648:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077701.203653:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617421440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077701.203677:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077701.203697:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077701.203706:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077701/real 1769077701] req@ffff9b0343deed80 x1855007617421568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077701.203715:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617421568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077701.203721:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077701.203723:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077701.203726:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077701.203730:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077701.203740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077701.203746:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077701/real 1769077701] req@ffff9b0343ded500 x1855007617421696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077701.203757:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617421696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077701.203762:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077701.203764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077701.203766:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077701.203770:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077701.203779:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077701.203785:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077701/real 1769077701] req@ffff9b0343def480 x1855007617421824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077701.203792:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617421824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077701.203797:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077701.203799:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077701.203801:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077701.203804:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077701.203813:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077701.203818:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077701/real 1769077701] req@ffff9b0343dee680 x1855007617421952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077701.203825:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617421952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077701.203834:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077701.203836:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077701.203838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617421952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077701.715301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077702.099344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077702.099344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077702.099432:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077702.739342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077703.123342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077703.123398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077703.123422:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077703.763342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077704.147352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077704.147352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077704.147409:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077704.787368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077705.171357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077705.171359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077705.171451:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077705.811346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077706.195388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077706.195388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077706.195465:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077706.323386:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077706.323402:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077706.323409:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077706.323421:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4695 00000100:00080000:0.0:1769077706.323443:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077706.323452:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077706.323508:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617422080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077706.323544:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077706.323549:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077706.323551:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077706.323556:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4695 00000100:00080000:0.0:1769077706.323560:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077706.323565:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077706.323577:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077706.323591:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617422208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077706.323604:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617422080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077706.323605:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077706.323615:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077706.323617:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077706.323622:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4695 00000100:00080000:0.0:1769077706.323626:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077706.323631:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077706.323636:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077706.323647:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617422336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077706.323656:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077706.323660:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077706.323662:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077706.323666:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4695 00000100:00080000:0.0:1769077706.323670:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077706.323674:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077706.323687:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769077706.323697:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077706.323699:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617422464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077706.323710:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077706.323714:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077706.323716:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077706.323720:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4695 00000100:00080000:0.0:1769077706.323724:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077706.323727:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077706.323731:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077706.323744:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617422592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077706.323750:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077706/real 1769077706] req@ffff9b0365a38a80 x1855007617422080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077706.323778:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617422080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077706.323791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077706.323798:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077706.323803:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617422080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077706.323833:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077706.323938:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077706.323950:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077706/real 1769077706] req@ffff9b0365a39c00 x1855007617422208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077706.323960:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617422208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077706.323967:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077706.323970:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077706.323973:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077706.323977:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077706.323990:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077706.323996:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077706/real 1769077706] req@ffff9b0365a38e00 x1855007617422336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077706.324007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617422336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077706.324016:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077706.324019:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077706.324022:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077706.324028:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077706.324049:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077706.324060:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077706/real 1769077706] req@ffff9b0365a3b480 x1855007617422464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077706.324069:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617422464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077706.324075:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077706.324077:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077706.324080:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077706.324084:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077706.324100:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077706.324111:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077706/real 1769077706] req@ffff9b0365a3aa00 x1855007617422592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077706.324124:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617422592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077706.324133:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077706.324136:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077706.324139:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077706.835362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077707.219390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077707.219490:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077707.219512:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077707.859321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077708.243340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077708.243368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077708.243442:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077708.883324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077709.267310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077709.267343:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077709.267357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077709.907394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077710.291343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077710.291343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077710.291399:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077710.931365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077711.315311:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077711.315320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077711.315386:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077711.443353:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077711.443362:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077711.443365:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077711.443373:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4700 00000100:00080000:3.0:1769077711.443382:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077711.443387:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077711.443415:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617422720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077711.443437:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077711.443442:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077711.443445:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077711.443450:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4700 00000100:00080000:3.0:1769077711.443455:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077711.443459:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077711.443467:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617422720:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769077711.443469:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077711.443488:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617422848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077711.443498:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077711.443501:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077711.443503:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077711.443506:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4700 00000100:00080000:3.0:1769077711.443509:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077711.443513:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077711.443517:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077711.443537:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617422976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077711.443543:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077711.443545:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077711.443546:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077711.443549:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4700 00000100:00080000:3.0:1769077711.443551:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077711.443553:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077711.443557:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077711.443567:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617423104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077711.443575:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077711.443578:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077711.443579:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077711.443582:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4700 00000100:00080000:3.0:1769077711.443586:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077711.443589:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422848:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769077711.443600:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077711.443607:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077711.443621:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617423232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077711.443633:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077711.443662:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077711.443689:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077711.955271:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077712.339343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077712.339343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077712.339409:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077712.979276:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077713.363341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077713.363429:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077713.363473:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077714.003273:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077714.387339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077714.387345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077714.387398:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769077714.515584:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769077714.515626:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769077714.515642:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077714.515677:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077714.515685:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077714.515693:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077714.515701:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769077714.515732:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077711/real 1769077714] req@ffff9b0352a19500 x1855007617423232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077714.515767:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617423232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077714.515777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077714.515782:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077714.515786:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077714.515793:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077711/real 1769077714] req@ffff9b0352a18380 x1855007617422848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077714.515805:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617422848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077714.515812:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077714.515815:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077714.515819:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077714.515825:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077711/real 1769077714] req@ffff9b0352a19c00 x1855007617422976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077714.515836:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617422976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077714.515843:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077714.515847:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077714.515850:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617422976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077714.515856:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077711/real 1769077714] req@ffff9b0352a1b800 x1855007617423104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077714.515870:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617423104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077714.515879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077714.515881:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077714.515885:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077714.515896:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077711/real 1769077714] req@ffff9b0352a18000 x1855007617422720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077714.515906:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617422720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077714.515913:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077714.515915:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077714.515918:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617422720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077715.027309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077715.411329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077715.411329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077715.411397:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077716.051318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077716.435341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077716.435341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077716.435406:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077716.563341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077716.563349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077716.563352:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077716.563360:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4705 00000100:00080000:3.0:1769077716.563375:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077716.563381:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077716.563409:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617423360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077716.563429:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077716.563433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077716.563434:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077716.563438:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4705 00000100:00080000:3.0:1769077716.563442:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077716.563446:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077716.563455:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077716.563469:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617423360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077716.563470:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617423488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077716.563487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077716.563490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077716.563492:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077716.563497:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4705 00000100:00080000:3.0:1769077716.563501:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077716.563505:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077716.563510:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077716.563527:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617423616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077716.563539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769077716.563540:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077716.563545:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077716.563547:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077716.563552:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4705 00000100:00080000:3.0:1769077716.563562:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077716.563566:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077716.563571:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077716.563582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617423744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077716.563589:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077716/real 1769077716] req@ffff9b0352a18000 x1855007617423360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077716.563594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077716.563601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077716.563605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077716.563622:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617423360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077716.563623:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4705 00000100:00080000:3.0:1769077716.563629:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077716.563636:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769077716.563640:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769077716.563642:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769077716.563647:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077716.563652:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617423360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077716.563658:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617423872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077716.563689:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077716.563714:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077716.563726:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077716/real 1769077716] req@ffff9b0352a1b800 x1855007617423488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077716.563749:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617423488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077716.563757:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077716.563760:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077716.563763:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077716.563769:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077716.563785:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077716.563794:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077716/real 1769077716] req@ffff9b0352a19c00 x1855007617423616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077716.563805:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617423616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077716.563812:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077716.563815:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077716.563818:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077716.563824:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077716.563838:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077716.563845:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077716/real 1769077716] req@ffff9b0352a18380 x1855007617423744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077716.563856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617423744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077716.563869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077716.563872:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077716.563875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077716.563880:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077716.563894:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077716.563901:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077716/real 1769077716] req@ffff9b0352a19500 x1855007617423872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077716.563912:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617423872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077716.563919:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077716.563922:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077716.563925:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617423872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077717.075319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077717.459349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077717.459395:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077717.459400:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077718.099316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077718.483356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077718.483372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077718.483415:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077719.123323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077719.507356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077719.507453:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077719.507477:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077720.147334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077720.531344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077720.531346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077720.531406:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077721.171320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077721.555364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077721.555364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077721.555468:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077721.683389:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077721.683400:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077721.683411:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077721.683420:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4710 00000100:00080000:0.0:1769077721.683430:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077721.683435:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077721.683465:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617424000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077721.683486:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077721.683489:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077721.683490:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077721.683493:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4710 00000100:00080000:0.0:1769077721.683495:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077721.683497:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077721.683504:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077721.683514:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617424128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077721.683522:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077721.683525:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077721.683526:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077721.683528:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4710 00000100:00080000:0.0:1769077721.683530:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077721.683534:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077721.683542:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077721.683543:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617424000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077721.683553:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617424256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077721.683560:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077721.683562:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077721.683563:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077721.683565:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4710 00000100:00080000:0.0:1769077721.683573:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077721.683575:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077721.683582:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077721.683607:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617424384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077721.683617:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077721.683621:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077721.683623:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077721.683626:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4710 00000100:00080000:0.0:1769077721.683630:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077721.683634:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077721.683639:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077721.683649:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617424512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769077721.683655:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077721.683717:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077721/real 1769077721] req@ffff9b0365a3b100 x1855007617424000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077721.683747:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617424000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077721.683758:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077721.683763:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077721.683768:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617424000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077721.683797:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077721.683821:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077721.683834:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077721/real 1769077721] req@ffff9b0365a39500 x1855007617424128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077721.683845:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617424128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077721.683854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077721.683857:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077721.683860:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077721.683866:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077721.683885:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077721.683896:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077721/real 1769077721] req@ffff9b0365a3bb80 x1855007617424256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077721.683908:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617424256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077721.683922:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077721.683925:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077721.683928:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077721.683934:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077721.683953:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077721.683964:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077721/real 1769077721] req@ffff9b0365a3ad80 x1855007617424384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077721.683984:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617424384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077721.683992:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077721.683996:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077721.683999:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077721.684005:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077721.684023:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077721.684034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077721/real 1769077721] req@ffff9b0365a39180 x1855007617424512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077721.684046:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617424512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077721.684054:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077721.684057:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077721.684060:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077722.195391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077722.579316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077722.579317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077722.579403:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077723.219287:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077723.603329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077723.603329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077723.603395:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077724.243339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077724.627333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077724.627360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077724.627388:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077725.267329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077725.651335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077725.651335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077725.651399:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077726.291351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077726.675318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077726.675318:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077726.675404:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077726.803331:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077726.803339:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077726.803342:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077726.803351:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4715 00000100:00080000:0.0:1769077726.803359:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077726.803364:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077726.803396:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617424640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077726.803417:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077726.803420:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077726.803421:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077726.803424:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4715 00000100:00080000:0.0:1769077726.803426:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077726.803428:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077726.803435:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077726.803448:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617424768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077726.803456:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077726.803458:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077726.803460:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769077726.803463:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617424640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077726.803464:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4715 00000100:00080000:0.0:1769077726.803468:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077726.803470:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077726.803474:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077726.803481:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617424896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077726.803490:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077726.803493:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077726.803495:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077726.803499:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4715 00000100:00080000:0.0:1769077726.803514:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077726.803519:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077726.803524:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077726.803535:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617425024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077726.803546:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769077726.803547:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077726.803552:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077726.803553:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077726.803557:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4715 00000100:00080000:0.0:1769077726.803560:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077726.803563:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077726.803567:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077726.803575:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617425152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077726.803591:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077726/real 1769077726] req@ffff9b0365a39180 x1855007617424640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077726.803610:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617424640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077726.803621:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077726.803625:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077726.803629:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617424640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077726.803658:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077726.803682:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077726.803694:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077726/real 1769077726] req@ffff9b0365a3ad80 x1855007617424768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077726.803707:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617424768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077726.803715:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077726.803719:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077726.803723:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077726.803729:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077726.803745:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077726.803755:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077726/real 1769077726] req@ffff9b0365a3bb80 x1855007617424896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077726.803765:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617424896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077726.803773:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077726.803776:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077726.803779:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617424896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077726.803784:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077726.803802:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077726.803812:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077726/real 1769077726] req@ffff9b0365a39500 x1855007617425024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077726.803824:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617425024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077726.803832:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077726.803835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077726.803838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077726.803844:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077726.803862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077726.803871:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077726/real 1769077726] req@ffff9b0365a3b100 x1855007617425152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077726.803882:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617425152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077726.803890:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077726.803893:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077726.803896:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077727.315331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769077727.699463:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077727.699608:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077727.699631:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077728.339339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077728.723301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077728.723302:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077728.723351:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077729.363408:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077729.747345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077729.747349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077729.747415:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077730.387327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077730.771330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077730.771422:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077730.771464:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077731.411346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077731.795351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077731.795415:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077731.795564:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:2.0:1769077731.923299:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077731.923309:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077731.923313:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077731.923322:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4721 00000100:00080000:2.0:1769077731.923330:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077731.923336:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077731.923363:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617425280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077731.923386:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077731.923389:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077731.923391:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077731.923395:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4721 00000100:00080000:2.0:1769077731.923399:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077731.923402:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077731.923411:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077731.923439:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617425280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077731.923448:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617425408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077731.923457:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077731.923459:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077731.923461:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077731.923464:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4721 00000100:00080000:2.0:1769077731.923467:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077731.923470:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077731.923474:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077731.923483:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617425536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077731.923490:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077731.923493:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077731.923493:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077731.923494:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077731.923497:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4721 00000100:00080000:2.0:1769077731.923500:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077731.923502:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077731.923506:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077731.923513:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617425664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077731.923524:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077731.923527:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077731/real 1769077731] req@ffff9b03482c4380 x1855007617425280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077731.923527:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077731.923529:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077731.923532:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4721 00000100:00080000:2.0:1769077731.923535:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077731.923538:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077731.923542:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077731.923543:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617425280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077731.923549:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617425792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077731.923553:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077731.923556:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077731.923560:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617425280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077731.923580:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077731.923600:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077731.923609:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077731/real 1769077731] req@ffff9b03482c5180 x1855007617425408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077731.923616:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617425408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077731.923621:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077731.923623:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077731.923625:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077731.923629:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077731.923638:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077731.923643:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077731/real 1769077731] req@ffff9b03482c4a80 x1855007617425536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077731.923649:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617425536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077731.923653:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077731.923655:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077731.923656:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077731.923659:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077731.923667:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077731.923672:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077731/real 1769077731] req@ffff9b03482c5500 x1855007617425664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077731.923678:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617425664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077731.923682:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077731.923684:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077731.923686:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077731.923688:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077731.923696:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077731.923700:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077731/real 1769077731] req@ffff9b03482c6300 x1855007617425792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077731.923705:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617425792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077731.923710:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077731.923711:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077731.923713:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617425792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077732.435346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077732.819341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077732.819341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077732.819358:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077733.459354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077733.843340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077733.843340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077733.843416:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077734.483317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077734.867296:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077734.867325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077734.867381:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077735.507343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077735.891305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077735.891310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077735.891382:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077736.531371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077736.915304:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077736.915304:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077736.915402:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077737.043359:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077737.043367:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077737.043382:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077737.043390:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4726 00000100:00080000:3.0:1769077737.043398:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077737.043404:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077737.043441:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617425920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077737.043470:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077737.043474:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077737.043477:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077737.043482:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4726 00000100:00080000:3.0:1769077737.043487:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077737.043492:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077737.043502:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077737.043506:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617425920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077737.043530:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617426048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077737.043540:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077737.043543:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077737.043544:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077737.043547:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4726 00000100:00080000:3.0:1769077737.043550:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077737.043554:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077737.043557:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077737.043566:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617426176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077737.043573:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769077737.043577:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077737.043578:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077737.043582:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077737.043586:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4726 00000100:00080000:3.0:1769077737.043589:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077737.043592:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077737.043596:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077737.043604:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617426304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077737.043610:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077737.043613:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077737.043616:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077737.043617:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9b0352a1ad80 x1855007617425920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077737.043622:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4726 00000100:00080000:3.0:1769077737.043627:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077737.043633:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077737.043636:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077737.043641:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617425920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077737.043647:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617426432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077737.043661:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077737.043669:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077737.043675:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617425920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077737.043695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077737.043717:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077737.043729:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9b0352a1b100 x1855007617426048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077737.043743:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617426048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077737.043753:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077737.043757:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077737.043761:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077737.043768:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077737.043784:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077737.043793:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9b0352a18a80 x1855007617426176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077737.043806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617426176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077737.043815:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077737.043818:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077737.043822:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077737.043828:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077737.043849:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077737.043860:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9b0352a1a680 x1855007617426304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077737.043868:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617426304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077737.043874:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077737.043875:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077737.043878:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077737.043882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077737.043903:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077737.043909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077737/real 1769077737] req@ffff9b0352a1b480 x1855007617426432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077737.043916:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617426432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077737.043921:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077737.043931:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077737.043933:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077737.555385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077737.939403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077737.939403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077737.939418:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077738.579302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077738.963371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077738.963422:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077738.963434:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077739.603367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077739.987362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077739.987362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077739.987442:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077740.627343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077741.011345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077741.011345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077741.011405:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077741.651350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077742.035330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077742.035330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077742.035391:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077742.163359:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077742.163368:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077742.163373:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077742.163381:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4731 00000100:00080000:3.0:1769077742.163390:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077742.163396:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077742.163428:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617426560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077742.163454:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077742.163457:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077742.163458:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077742.163461:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4731 00000100:00080000:3.0:1769077742.163464:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077742.163466:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077742.163473:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077742.163482:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617426688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077742.163489:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077742.163492:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077742.163494:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077742.163496:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4731 00000100:00080000:3.0:1769077742.163499:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077742.163501:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077742.163505:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077742.163508:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617426560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077742.163514:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617426816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077742.163523:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077742.163525:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077742.163526:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077742.163529:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4731 00000100:00080000:3.0:1769077742.163532:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077742.163534:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077742.163537:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077742.163545:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617426944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077742.163551:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077742.163553:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077742.163554:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077742.163557:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4731 00000100:00080000:3.0:1769077742.163559:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077742.163561:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077742.163564:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077742.163570:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617427072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077742.163587:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077742.163623:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077742/real 1769077742] req@ffff9b0352a1aa00 x1855007617426560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077742.163637:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617426560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077742.163645:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077742.163648:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077742.163651:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617426560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077742.163657:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077742.163671:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077742.163679:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077742/real 1769077742] req@ffff9b0352a18700 x1855007617426688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077742.163687:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617426688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077742.163693:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077742.163695:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077742.163697:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077742.163715:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077742.163725:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077742.163731:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077742/real 1769077742] req@ffff9b0352a19f80 x1855007617426816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077742.163738:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617426816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077742.163744:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077742.163745:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077742.163747:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077742.163751:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077742.163761:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077742.163767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077742/real 1769077742] req@ffff9b0352a1a300 x1855007617426944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077742.163774:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617426944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077742.163779:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077742.163781:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077742.163783:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617426944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077742.163786:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077742.163795:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077742.163800:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077742/real 1769077742] req@ffff9b0352a18e00 x1855007617427072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077742.163806:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617427072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077742.163810:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077742.163812:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077742.163815:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077742.675349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077743.059341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077743.059342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077743.059363:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077743.699316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077744.083316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077744.083316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077744.083387:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077744.723347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077745.107336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077745.107337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077745.107394:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077745.747310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077746.131310:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077746.131368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077746.131387:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077746.771342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077747.155348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077747.155348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077747.155420:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077747.283343:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077747.283351:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077747.283355:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077747.283363:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4736 00000100:00080000:2.0:1769077747.283371:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077747.283378:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077747.283410:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617427200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077747.283440:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077747.283445:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077747.283448:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077747.283461:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4736 00000100:00080000:2.0:1769077747.283466:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077747.283472:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077747.283478:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617427200:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769077747.283484:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077747.283503:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617427328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077747.283517:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077747.283521:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077747.283524:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077747.283529:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4736 00000100:00080000:2.0:1769077747.283534:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077747.283539:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077747.283545:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1769077747.283550:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077747.283560:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617427456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077747.283579:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077747.283583:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077747.283585:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077747.283590:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4736 00000100:00080000:2.0:1769077747.283595:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077747.283601:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077747.283606:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077747.283617:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617427584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077747.283627:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077747.283631:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077747.283633:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077747.283637:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4736 00000100:00080000:2.0:1769077747.283640:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077747.283644:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077747.283649:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077747.283652:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077747/real 1769077747] req@ffff9b03482c6d80 x1855007617427200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077747.283671:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617427712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077747.283673:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617427200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077747.283777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077747.283782:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077747.283785:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617427200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077747.283840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077747.283862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077747.283877:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077747/real 1769077747] req@ffff9b03482c6680 x1855007617427328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077747.283887:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617427328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077747.283893:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077747.283896:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077747.283898:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077747.283904:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077747.283914:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077747.283920:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077747/real 1769077747] req@ffff9b03482c7100 x1855007617427456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077747.283928:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617427456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077747.283933:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077747.283934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077747.283936:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077747.283940:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077747.283950:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077747.283956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077747/real 1769077747] req@ffff9b03482c4000 x1855007617427584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077747.283963:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617427584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077747.283968:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077747.283970:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077747.283972:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077747.283975:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077747.283984:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077747.283989:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077747/real 1769077747] req@ffff9b03482c6a00 x1855007617427712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077747.283996:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617427712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077747.284001:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077747.284003:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077747.284004:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077747.795341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077748.179289:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077748.179311:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077748.179347:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077748.819339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077749.203374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077749.203384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077749.203449:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077749.843392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077750.227311:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077750.227364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077750.227376:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077750.867295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077751.251392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077751.251392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077751.251454:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077751.891329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077752.275338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077752.275345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077752.275364:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077752.403445:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077752.403454:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077752.403458:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077752.403466:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4741 00000100:00080000:1.0:1769077752.403475:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077752.403480:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077752.403511:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617427840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077752.403533:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077752.403536:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077752.403538:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077752.403541:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4741 00000100:00080000:1.0:1769077752.403544:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077752.403547:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077752.403554:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077752.403563:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617427968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077752.403569:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617427840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077752.403574:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077752.403577:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077752.403578:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077752.403581:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4741 00000100:00080000:1.0:1769077752.403584:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077752.403587:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077752.403591:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077752.403600:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617428096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077752.403606:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077752.403609:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077752.403610:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077752.403613:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4741 00000100:00080000:1.0:1769077752.403615:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077752.403618:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077752.403621:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077752.403628:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617428224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077752.403632:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077752.403641:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077752.403644:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077752.403645:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077752.403649:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4741 00000100:00080000:1.0:1769077752.403652:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077752.403663:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077752.403668:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077752.403671:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077752/real 1769077752] req@ffff9b0343defb80 x1855007617427840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077752.403683:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617428352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077752.403690:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617427840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077752.403714:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077752.403719:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077752.403724:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617427840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077752.403744:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077752.403759:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077752.403767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077752/real 1769077752] req@ffff9b0343dece00 x1855007617427968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077752.403775:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617427968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077752.403781:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077752.403783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077752.403785:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617427968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077752.403789:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077752.403798:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077752.403803:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077752/real 1769077752] req@ffff9b0343dee680 x1855007617428096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077752.403810:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617428096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077752.403816:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077752.403817:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077752.403819:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077752.403823:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077752.403833:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077752.403839:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077752/real 1769077752] req@ffff9b0343def480 x1855007617428224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077752.403846:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617428224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077752.403851:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077752.403853:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077752.403855:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077752.403858:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077752.403867:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077752.403872:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077752/real 1769077752] req@ffff9b0343ded500 x1855007617428352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077752.403878:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617428352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077752.403883:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077752.403885:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077752.403886:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077752.915336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077753.299337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077753.299358:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077753.299374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077753.939351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077754.323345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077754.323345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077754.323410:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077754.963338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077755.347334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077755.347334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077755.347399:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077755.987330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077756.371351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077756.371351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077756.371425:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077757.011377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077757.395334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077757.395345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077757.395376:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077757.523387:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077757.523398:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077757.523403:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077757.523413:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4746 00000100:00080000:1.0:1769077757.523433:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077757.523444:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077757.523482:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617428480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077757.523515:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077757.523518:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077757.523520:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077757.523526:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4746 00000100:00080000:1.0:1769077757.523530:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077757.523534:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077757.523545:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077757.523563:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617428480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077757.523564:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617428608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077757.523582:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077757.523586:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077757.523588:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077757.523594:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4746 00000100:00080000:1.0:1769077757.523598:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077757.523602:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077757.523607:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077757.523620:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617428736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077757.523630:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077757.523633:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077757.523636:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769077757.523640:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077757.523645:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4746 00000100:00080000:1.0:1769077757.523651:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077757.523657:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077757.523663:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077757.523673:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617428864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077757.523685:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077757.523688:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077757.523690:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077757.523694:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4746 00000100:00100000:3.0:1769077757.523698:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077757/real 1769077757] req@ffff9b0343deed80 x1855007617428480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077757.523700:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077757.523709:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077757.523717:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077757.523726:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617428480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077757.523731:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617428992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077757.523748:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077757.523756:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077757.523762:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617428480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077757.523790:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077757.523811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077757.523829:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077757/real 1769077757] req@ffff9b0343deca80 x1855007617428608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077757.523839:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617428608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077757.523845:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077757.523848:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077757.523850:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077757.523855:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077757.523867:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077757.523873:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077757/real 1769077757] req@ffff9b0343deea00 x1855007617428736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077757.523881:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617428736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077757.523886:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077757.523888:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077757.523890:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077757.523894:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077757.523904:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077757.523909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077757/real 1769077757] req@ffff9b0343def800 x1855007617428864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077757.523917:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617428864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077757.523922:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077757.523923:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077757.523925:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077757.523929:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077757.523938:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077757.523943:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077757/real 1769077757] req@ffff9b0343ded880 x1855007617428992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077757.523950:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617428992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077757.523959:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077757.523962:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077757.523964:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617428992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077758.035307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077758.419422:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077758.419477:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077758.419727:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077759.059343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077759.443476:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077759.443495:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077759.443593:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077760.083313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077760.467507:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077760.467603:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077760.467658:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077761.107347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077761.491560:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077761.491621:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077761.492171:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077762.131302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769077762.515616:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077762.516091:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077762.516169:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769077762.643337:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077762.643345:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077762.643350:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077762.643358:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4751 00000100:00080000:3.0:1769077762.643365:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077762.643371:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077762.643402:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617429120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077762.643429:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077762.643432:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077762.643434:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077762.643439:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4751 00000100:00080000:3.0:1769077762.643443:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077762.643446:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077762.643457:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077762.643469:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617429248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077762.643479:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617429120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077762.643480:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077762.643484:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077762.643485:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077762.643489:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4751 00000100:00080000:3.0:1769077762.643492:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077762.643495:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077762.643500:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077762.643509:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617429376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077762.643516:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077762.643519:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077762.643521:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077762.643524:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4751 00000100:00080000:3.0:1769077762.643527:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077762.643530:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077762.643534:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769077762.643538:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077762.643542:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617429504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077762.643550:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077762.643552:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077762.643555:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077762.643559:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4751 00000100:00080000:3.0:1769077762.643561:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077762.643564:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077762.643568:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077762.643574:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077762/real 1769077762] req@ffff9b0352a18e00 x1855007617429120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077762.643577:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617429632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077762.643592:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617429120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077762.643602:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077762.643607:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077762.643611:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617429120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077762.643635:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617429248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077762.643654:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077762.643663:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077762/real 1769077762] req@ffff9b0352a1a300 x1855007617429248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077762.643671:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617429248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077762.643677:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077762.643679:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077762.643682:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617429248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077762.643685:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617429376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077762.643697:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077762.643704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077762/real 1769077762] req@ffff9b0352a19f80 x1855007617429376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077762.643716:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617429376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077762.643724:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077762.643728:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077762.643731:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617429376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077762.643737:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617429504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077762.643757:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077762.643765:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077762/real 1769077762] req@ffff9b0352a18700 x1855007617429504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077762.643773:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617429504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077762.643779:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077762.643781:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077762.643783:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617429504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077762.643786:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617429632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077762.643796:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077762.643801:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077762/real 1769077762] req@ffff9b0352a1aa00 x1855007617429632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077762.643813:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617429632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077762.643818:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077762.643819:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077762.643821:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617429632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077763.155375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077763.539766:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077763.539916:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077763.539928:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077764.179337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077764.563355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077764.563362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077764.563442:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077765.203343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077765.587300:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077765.587329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077765.587362:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077766.227367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077766.611382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077766.611385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077766.611474:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077767.251342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077767.635309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077767.635386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077767.635401:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077767.763367:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077767.763377:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077767.763382:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077767.763392:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4756 00000100:00080000:1.0:1769077767.763401:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077767.763409:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077767.763448:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617429760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077767.763481:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077767.763485:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077767.763487:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077767.763493:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4756 00000100:00080000:1.0:1769077767.763497:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077767.763501:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077767.763513:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077767.763528:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617429888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077767.763531:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617429760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077767.763550:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077767.763555:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077767.763557:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077767.763563:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4756 00000100:00080000:1.0:1769077767.763567:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077767.763572:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077767.763578:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077767.763591:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617430016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769077767.763599:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077767.763608:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077767.763612:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077767.763615:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077767.763621:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4756 00000100:00080000:1.0:1769077767.763625:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077767.763630:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077767.763636:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077767.763645:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077767/real 1769077767] req@ffff9b0343dedf80 x1855007617429760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077767.763651:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617430144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077767.763673:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769077767.763678:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617429760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077767.763679:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077767.763684:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077767.763695:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077767.763695:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4756 00000100:00080000:0.0:1769077767.763703:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769077767.763703:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769077767.763710:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617429760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077767.763713:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077767.763721:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077767.763733:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617430272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077767.763735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617429888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077767.763762:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077767.763773:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077767/real 1769077767] req@ffff9b0343def100 x1855007617429888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077767.763783:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617429888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077767.763795:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077767.763797:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077767.763799:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617429888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077767.763804:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077767.763839:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077767.763848:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077767/real 1769077767] req@ffff9b0343dee300 x1855007617430016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077767.763857:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617430016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077767.763863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077767.763865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077767.763867:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077767.763870:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077767.763883:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077767.763889:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077767/real 1769077767] req@ffff9b0343dec380 x1855007617430144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077767.763896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617430144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077767.763900:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077767.763902:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077767.763905:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077767.763915:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077767.763924:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077767.763930:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077767/real 1769077767] req@ffff9b0343ded180 x1855007617430272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077767.763937:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617430272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077767.763942:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077767.763944:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077767.763946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077768.275348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077768.659344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077768.659344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077768.659376:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077769.299347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077769.683338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077769.683338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077769.683394:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077770.323351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077770.707297:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077770.707297:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077770.707371:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077771.347340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077771.731334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077771.731334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077771.731452:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077772.371345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077772.755350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077772.755371:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077772.755426:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077772.883351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077772.883360:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077772.883364:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077772.883372:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4762 00000100:00080000:3.0:1769077772.883380:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077772.883386:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077772.883418:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617430400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077772.883441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077772.883444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077772.883445:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077772.883449:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4762 00000100:00080000:3.0:1769077772.883451:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077772.883453:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077772.883460:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077772.883469:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617430528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077772.883477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077772.883479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077772.883481:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077772.883487:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4762 00000100:00100000:0.0:1769077772.883489:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617430400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077772.883489:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077772.883495:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077772.883501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077772.883509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617430656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077772.883517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077772.883519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077772.883521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077772.883524:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4762 00000100:00080000:3.0:1769077772.883526:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077772.883528:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077772.883532:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077772.883538:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617430784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077772.883544:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077772.883547:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077772.883548:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077772.883550:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4762 00000100:00080000:3.0:1769077772.883553:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769077772.883558:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077772.883559:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077772.883565:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077772.883572:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617430912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077772.883605:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077772/real 1769077772] req@ffff9b0352a1b480 x1855007617430400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077772.883620:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617430400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077772.883628:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077772.883631:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077772.883634:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617430400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077772.883640:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077772.883658:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077772.883667:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077772/real 1769077772] req@ffff9b0352a1a680 x1855007617430528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077772.883676:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617430528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077772.883682:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077772.883686:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077772.883689:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077772.883708:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769077772.883719:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077772.883741:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077772/real 1769077772] req@ffff9b0352a18a80 x1855007617430656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077772.883752:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617430656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077772.883758:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077772.883761:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077772.883763:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077772.883767:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077772.883789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077772.883799:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077772/real 1769077772] req@ffff9b0352a1b100 x1855007617430784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077772.883811:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617430784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077772.883819:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077772.883821:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077772.883824:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077772.883829:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769077772.883839:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077772.883845:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077772/real 1769077772] req@ffff9b0352a1ad80 x1855007617430912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077772.883852:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617430912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077772.883857:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077772.883859:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077772.883861:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617430912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077773.395359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077773.779344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077773.779344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077773.779370:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077774.419338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077774.803349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077774.803387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077774.803415:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077775.443342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077775.827338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077775.827338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077775.827408:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077776.467354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077776.851335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077776.851335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077776.851399:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077777.491310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077777.875299:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077777.875301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077777.875395:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077778.003346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077778.003357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077778.003362:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077778.003372:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4767 00000100:00080000:3.0:1769077778.003382:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077778.003389:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077778.003423:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617431040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077778.003447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077778.003451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077778.003453:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077778.003457:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4767 00000100:00080000:3.0:1769077778.003462:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077778.003467:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077778.003476:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077778.003491:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617431040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077778.003492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617431168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077778.003509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077778.003513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077778.003516:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077778.003521:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4767 00000100:00080000:3.0:1769077778.003524:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077778.003528:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077778.003534:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077778.003545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617431296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077778.003555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077778.003559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077778.003561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077778.003565:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4767 00000100:00080000:3.0:1769077778.003569:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077778.003573:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077778.003578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077778.003590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617431424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077778.003598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077778.003602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077778.003604:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769077778.003609:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431168:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769077778.003611:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4767 00000100:00080000:3.0:1769077778.003619:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077778.003626:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077778.003631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077778.003661:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617431552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769077778.003680:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077778.003706:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077778.003734:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077778.515277:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077778.899356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077778.899356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077778.899420:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077779.539315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077779.923385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077779.923399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077779.923472:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077780.563294:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077780.947328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077780.947328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077780.947391:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769077781.075475:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769077781.075511:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077781.075525:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077781.075557:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077781.075567:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077781.075575:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077781.075583:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769077781.075604:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077781] req@ffff9b0352a18380 x1855007617431168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077781.075630:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617431168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077781.075643:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077781.075649:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077781.075654:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077781.075662:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077781] req@ffff9b0352a19c00 x1855007617431296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077781.075676:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617431296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077781.075689:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077781.075693:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077781.075697:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077781.075704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077781] req@ffff9b0352a1b800 x1855007617431424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077781.075715:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617431424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077781.075723:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077781.075727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077781.075730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077781.075737:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077781] req@ffff9b0352a19500 x1855007617431040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077781.075749:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617431040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077781.075758:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077781.075761:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077781.075765:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617431040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077781.075790:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077778/real 1769077781] req@ffff9b0352a18000 x1855007617431552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077781.075800:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617431552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077781.075809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769077781.075813:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077781.075816:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077781.587299:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077781.971338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077781.971384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077781.971645:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077782.611392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077782.995308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077782.995310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077782.995380:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077783.123410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077783.123419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077783.123423:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077783.123431:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4772 00000100:00080000:0.0:1769077783.123441:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077783.123448:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077783.123486:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617431680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077783.123524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077783.123529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077783.123531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077783.123536:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4772 00000100:00080000:0.0:1769077783.123544:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077783.123551:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077783.123575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077783.123592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617431808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077783.123605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077783.123610:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617431680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077783.123611:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077783.123616:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077783.123624:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4772 00000100:00080000:0.0:1769077783.123628:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077783.123633:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077783.123639:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077783.123652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617431936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077783.123662:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077783.123666:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077783.123668:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077783.123673:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4772 00000100:00080000:0.0:1769077783.123680:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077783.123685:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077783.123693:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077783.123701:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077783.123705:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617432064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077783.123717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077783.123721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077783.123723:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077783.123728:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4772 00000100:00080000:0.0:1769077783.123732:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077783.123736:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077783.123741:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077783.123752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077783/real 1769077783] req@ffff9b0365a3aa00 x1855007617431680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077783.123755:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617432192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077783.123798:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617431680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077783.123810:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077783.123816:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077783.123821:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617431680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077783.123849:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077783.123876:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077783.123890:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077783/real 1769077783] req@ffff9b0365a3b480 x1855007617431808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077783.123903:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617431808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077783.123914:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077783.123917:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077783.123921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077783.123928:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077783.123949:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077783.123962:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077783/real 1769077783] req@ffff9b0365a38e00 x1855007617431936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077783.123973:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617431936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077783.123980:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077783.123983:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077783.123987:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617431936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077783.123993:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077783.124013:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077783.124024:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077783/real 1769077783] req@ffff9b0365a39c00 x1855007617432064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077783.124037:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617432064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077783.124045:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077783.124048:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077783.124053:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077783.124059:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077783.124079:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077783.124090:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077783/real 1769077783] req@ffff9b0365a38a80 x1855007617432192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077783.124103:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617432192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077783.124112:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077783.124115:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077783.124119:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077783.635295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077784.019358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077784.019364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077784.019408:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077784.659360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077785.043334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077785.043374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077785.043407:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077785.683321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077786.067382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077786.067382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077786.067480:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077786.707310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077787.091310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077787.091311:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077787.091412:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077787.731388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077788.115350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077788.115350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077788.115458:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077788.243344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077788.243354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077788.243358:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077788.243366:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4777 00000100:00080000:0.0:1769077788.243375:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077788.243382:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077788.243417:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617432320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077788.243442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077788.243446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077788.243448:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077788.243451:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4777 00000100:00080000:0.0:1769077788.243455:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077788.243459:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077788.243485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077788.243514:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617432448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077788.243519:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617432320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077788.243535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077788.243539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077788.243542:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077788.243548:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4777 00000100:00080000:0.0:1769077788.243552:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077788.243558:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077788.243565:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077788.243578:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617432576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077788.243585:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077788.243594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077788.243597:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077788.243599:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077788.243603:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4777 00000100:00080000:0.0:1769077788.243607:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077788.243611:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077788.243616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077788.243621:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077788/real 1769077788] req@ffff9b0365a38a80 x1855007617432320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077788.243628:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617432704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077788.243653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077788.243657:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077788.243658:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077788.243662:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4777 00000100:00080000:0.0:1769077788.243664:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077788.243667:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617432320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077788.243670:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077788.243676:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1769077788.243681:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1769077788.243687:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617432832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077788.243687:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077788.243696:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617432320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077788.243721:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077788.243737:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077788.243745:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077788/real 1769077788] req@ffff9b0365a39c00 x1855007617432448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077788.243753:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617432448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077788.243759:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077788.243761:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077788.243763:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077788.243767:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077788.243776:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077788.243781:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077788/real 1769077788] req@ffff9b0365a38e00 x1855007617432576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077788.243788:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617432576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077788.243797:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077788.243800:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077788.243802:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077788.243805:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077788.243815:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077788.243820:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077788/real 1769077788] req@ffff9b0365a3b480 x1855007617432704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077788.243826:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617432704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077788.243831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077788.243833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077788.243835:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077788.243838:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077788.243847:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077788.243852:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077788/real 1769077788] req@ffff9b0365a3aa00 x1855007617432832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077788.243859:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617432832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077788.243863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077788.243865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077788.243868:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617432832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077788.755359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077789.139315:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077789.139383:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077789.139400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077789.779344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077790.163351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077790.163377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077790.163416:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077790.803385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077791.187354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077791.187354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077791.187477:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077791.827358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077792.211364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077792.211364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077792.211461:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077792.851320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077793.235397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077793.235409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077793.235486:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077793.363379:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077793.363417:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077793.363423:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077793.363434:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4782 00000100:00080000:2.0:1769077793.363447:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077793.363454:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077793.363498:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617432960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077793.363538:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077793.363543:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077793.363546:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077793.363552:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4782 00000100:00080000:2.0:1769077793.363556:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077793.363562:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077793.363575:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077793.363591:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617433088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077793.363603:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077793.363608:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1769077793.363611:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617432960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077793.363612:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077793.363618:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4782 00000100:00080000:2.0:1769077793.363625:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077793.363629:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077793.363633:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077793.363643:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617433216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077793.363650:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077793.363653:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077793.363654:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077793.363657:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4782 00000100:00080000:2.0:1769077793.363660:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077793.363663:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077793.363667:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077793.363675:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617433344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077793.363682:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077793.363684:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077793.363686:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077793.363689:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4782 00000100:00080000:2.0:1769077793.363693:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077793.363695:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077793.363700:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077793.363712:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077793.363713:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617433472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077793.363777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077793/real 1769077793] req@ffff9b03482c4700 x1855007617432960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077793.363804:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617432960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077793.363817:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077793.363824:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077793.363829:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617432960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077793.363862:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077793.363892:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077793.363910:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077793/real 1769077793] req@ffff9b03482c4e00 x1855007617433088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077793.363928:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617433088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077793.363939:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077793.363944:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077793.363948:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077793.363956:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077793.363978:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077793.363991:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077793/real 1769077793] req@ffff9b03482c7b80 x1855007617433216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077793.364007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617433216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077793.364017:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077793.364022:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077793.364026:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077793.364034:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077793.364057:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077793.364070:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077793/real 1769077793] req@ffff9b03482c5f80 x1855007617433344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077793.364085:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617433344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077793.364095:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077793.364099:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077793.364104:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077793.364112:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077793.364133:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077793.364146:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077793/real 1769077793] req@ffff9b03482c5c00 x1855007617433472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077793.364162:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617433472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077793.364172:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077793.364176:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077793.364181:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077793.875303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077794.259332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077794.259332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077794.259409:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077794.899331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077795.283349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077795.283349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077795.283419:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077795.923377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077796.307385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077796.307420:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077796.307451:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077796.947382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077797.331409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077797.331418:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077797.331461:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077797.971382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077798.355378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077798.355391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077798.355463:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077798.483411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077798.483429:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077798.483435:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077798.483451:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4787 00000100:00080000:0.0:1769077798.483464:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077798.483472:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077798.483517:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617433600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077798.483547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077798.483552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077798.483554:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077798.483560:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4787 00000100:00080000:0.0:1769077798.483565:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077798.483570:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077798.483583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077798.483600:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617433728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077798.483613:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617433600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077798.483614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077798.483625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077798.483627:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077798.483633:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4787 00000100:00080000:0.0:1769077798.483637:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077798.483643:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077798.483649:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077798.483663:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617433856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077798.483673:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077798.483676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077798.483677:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077798.483680:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4787 00000100:00080000:0.0:1769077798.483692:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077798.483696:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077798.483701:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077798.483710:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077798.483713:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617433984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077798.483727:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077798.483729:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077798.483731:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077798.483734:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4787 00000100:00080000:0.0:1769077798.483738:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077798.483740:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077798.483743:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077798.483751:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617434112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077798.483774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077798/real 1769077798] req@ffff9b0365a3b100 x1855007617433600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077798.483808:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617433600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077798.483820:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077798.483825:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077798.483830:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617433600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077798.483862:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077798.483885:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077798.483895:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077798/real 1769077798] req@ffff9b0365a39500 x1855007617433728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077798.483905:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617433728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077798.483912:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077798.483916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077798.483920:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077798.483926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077798.483946:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077798.483958:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077798/real 1769077798] req@ffff9b0365a3bb80 x1855007617433856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077798.483971:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617433856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077798.483987:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077798.483991:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077798.483994:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077798.484001:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077798.484023:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077798.484035:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077798/real 1769077798] req@ffff9b0365a3ad80 x1855007617433984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077798.484049:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617433984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077798.484058:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077798.484061:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077798.484064:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617433984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077798.484069:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617434112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077798.484084:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077798.484092:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077798/real 1769077798] req@ffff9b0365a39180 x1855007617434112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077798.484100:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617434112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077798.484105:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077798.484107:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077798.484110:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617434112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077798.995347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077799.379354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077799.379354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077799.379417:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077800.019350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077800.403343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077800.403365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077800.403424:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077801.043330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077801.427392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077801.427392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077801.427545:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077802.067430:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077802.451313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077802.451347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077802.451497:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077803.091344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077803.475342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077803.475381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077803.475405:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077803.603329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077803.603339:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077803.603352:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077803.603363:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4792 00000100:00080000:0.0:1769077803.603373:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077803.603380:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077803.603419:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617434240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077803.603456:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077803.603463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077803.603466:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077803.603471:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4792 00000100:00080000:0.0:1769077803.603475:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077803.603479:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077803.603489:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077803.603504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617434368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077803.603515:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617434240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077803.603517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077803.603528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077803.603530:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077803.603536:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4792 00000100:00080000:0.0:1769077803.603538:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077803.603542:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077803.603547:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077803.603556:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617434496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077803.603563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077803.603566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077803.603567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077803.603569:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4792 00000100:00080000:0.0:1769077803.603572:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077803.603574:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077803.603578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077803.603586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617434624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077803.603589:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077803.603598:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077803.603602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077803.603604:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077803.603608:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4792 00000100:00080000:0.0:1769077803.603611:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077803.603613:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077803.603616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077803.603625:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617434752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077803.603630:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077803/real 1769077803] req@ffff9b0365a39180 x1855007617434240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077803.603650:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617434240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077803.603659:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077803.603663:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077803.603666:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617434240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077803.603684:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617434368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077803.603700:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077803.603707:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077803/real 1769077803] req@ffff9b0365a3ad80 x1855007617434368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077803.603716:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617434368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077803.603722:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077803.603725:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077803.603727:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617434368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077803.603730:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617434496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077803.603740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077803.603746:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077803/real 1769077803] req@ffff9b0365a3bb80 x1855007617434496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077803.603753:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617434496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077803.603758:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077803.603760:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077803.603762:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617434496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077803.603765:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617434624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077803.603775:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077803.603781:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077803/real 1769077803] req@ffff9b0365a39500 x1855007617434624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077803.603788:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617434624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077803.603794:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077803.603796:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077803.603800:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617434624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077803.603806:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617434752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077803.603826:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077803.603837:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077803/real 1769077803] req@ffff9b0365a3b100 x1855007617434752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077803.603850:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617434752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077803.603859:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077803.603862:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077803.603864:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617434752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077804.115328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077804.499346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077804.499376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077804.499457:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077805.139353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077805.523329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077805.523330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077805.523365:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077806.163322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077806.547292:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077806.547327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077806.547365:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077807.187338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077807.571346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077807.571346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077807.571404:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077808.211338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077808.595361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077808.595361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077808.595444:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077808.723391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077808.723403:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077808.723407:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077808.723419:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4797 00000100:00080000:1.0:1769077808.723429:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077808.723437:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077808.723479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617434880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077808.723509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077808.723514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077808.723516:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077808.723522:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4797 00000100:00080000:1.0:1769077808.723526:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077808.723530:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077808.723541:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077808.723557:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617435008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077808.723569:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617434880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077808.723572:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077808.723581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077808.723583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077808.723589:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4797 00000100:00080000:1.0:1769077808.723594:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077808.723599:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077808.723605:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077808.723620:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617435136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077808.723631:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077808.723635:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077808.723637:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769077808.723642:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077808.723644:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4797 00000100:00080000:1.0:1769077808.723651:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077808.723657:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077808.723664:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077808.723677:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617435264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077808.723688:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077808/real 1769077808] req@ffff9b0343dec000 x1855007617434880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077808.723689:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077808.723699:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077808.723704:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077808.723714:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4797 00000100:00080000:1.0:1769077808.723722:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077808.723726:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077808.723733:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077808.723748:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617434880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077808.723750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617435392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077808.723775:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077808.723783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077808.723788:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617434880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077808.723813:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077808.723838:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077808.723850:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077808/real 1769077808] req@ffff9b0343dec700 x1855007617435008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077808.723865:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617435008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077808.723875:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077808.723878:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077808.723882:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077808.723888:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077808.723906:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077808.723917:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077808/real 1769077808] req@ffff9b0343ded180 x1855007617435136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077808.723929:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617435136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077808.723944:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077808.723947:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077808.723951:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077808.723957:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077808.723976:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077808.723987:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077808/real 1769077808] req@ffff9b0343dec380 x1855007617435264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077808.724000:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617435264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077808.724009:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077808.724013:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077808.724016:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077808.724022:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077808.724039:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077808.724050:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077808/real 1769077808] req@ffff9b0343dee300 x1855007617435392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077808.724063:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617435392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077808.724071:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077808.724074:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077808.724079:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077809.235359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077809.619344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077809.619344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077809.619410:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077810.259346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077810.643362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077810.643378:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077810.643393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077811.283359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077811.667349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077811.667349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077811.667409:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077812.307333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077812.691347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077812.691348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077812.691436:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077813.331360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077813.715337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077813.715337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077813.715409:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077813.843334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077813.843343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077813.843347:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077813.843356:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4803 00000100:00080000:1.0:1769077813.843364:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077813.843370:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077813.843401:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617435520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077813.843430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077813.843432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077813.843434:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077813.843437:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4803 00000100:00080000:1.0:1769077813.843439:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077813.843442:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077813.843448:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077813.843457:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617435648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077813.843465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077813.843468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077813.843469:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077813.843472:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4803 00000100:00080000:1.0:1769077813.843475:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077813.843480:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617435520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769077813.843481:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077813.843488:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077813.843495:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617435776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077813.843503:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077813.843505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077813.843506:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077813.843509:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4803 00000100:00080000:1.0:1769077813.843512:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077813.843515:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077813.843518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077813.843526:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617435904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077813.843533:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077813.843535:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077813.843536:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077813.843538:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4803 00000100:00080000:1.0:1769077813.843542:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769077813.843546:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077813.843548:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077813.843553:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077813.843561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617436032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077813.843584:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077813/real 1769077813] req@ffff9b0343def100 x1855007617435520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077813.843600:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617435520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077813.843607:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077813.843610:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077813.843613:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617435520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077813.843619:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077813.843633:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077813.843641:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077813/real 1769077813] req@ffff9b0343dedf80 x1855007617435648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077813.843649:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617435648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077813.843655:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077813.843658:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077813.843660:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077813.843677:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077813.843688:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077813.843693:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077813/real 1769077813] req@ffff9b0343ded880 x1855007617435776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077813.843700:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617435776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077813.843706:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077813.843708:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077813.843710:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077813.843713:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077813.843722:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077813.843727:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077813/real 1769077813] req@ffff9b0343def800 x1855007617435904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077813.843734:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617435904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077813.843739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077813.843741:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077813.843744:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617435904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077813.843747:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077813.843757:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077813.843762:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077813/real 1769077813] req@ffff9b0343deea00 x1855007617436032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077813.843769:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617436032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077813.843774:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077813.843775:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077813.843777:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077814.355320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077814.739311:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077814.739311:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077814.739370:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077815.379342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077815.763346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077815.763366:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077815.763466:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077816.403359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077816.787345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077816.787360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077816.787491:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769077817.427414:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077817.811385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077817.811457:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077817.811464:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077818.451358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077818.835399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077818.835429:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077818.835490:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077818.963468:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077818.963484:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077818.963490:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077818.963505:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4808 00000100:00080000:0.0:1769077818.963517:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077818.963527:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077818.963572:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617436160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077818.963609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077818.963614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077818.963616:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077818.963623:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4808 00000100:00080000:0.0:1769077818.963628:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077818.963634:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077818.963646:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077818.963646:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617436160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077818.963667:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617436288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077818.963680:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077818.963684:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077818.963687:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077818.963692:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4808 00000100:00080000:0.0:1769077818.963717:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077818.963722:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077818.963728:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077818.963733:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077818.963746:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617436416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077818.963759:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077818.963763:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077818.963766:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077818.963772:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4808 00000100:00080000:0.0:1769077818.963777:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077818.963782:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077818.963786:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077818/real 1769077818] req@ffff9b0365a3aa00 x1855007617436160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077818.963791:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077818.963810:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617436544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077818.963816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617436160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077818.963823:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077818.963826:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077818.963828:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077818.963832:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4808 00000100:00080000:0.0:1769077818.963835:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077818.963837:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077818.963841:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077818.963848:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617436672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077818.963879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077818.963884:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077818.963888:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617436160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077818.963926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077818.963967:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077818.963995:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077818/real 1769077818] req@ffff9b0365a3b480 x1855007617436288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077818.964010:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617436288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077818.964038:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077818.964044:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077818.964062:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077818.964070:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077818.964104:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077818.964129:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077818/real 1769077818] req@ffff9b0365a38e00 x1855007617436416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077818.964147:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617436416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077818.964173:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077818.964175:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077818.964178:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077818.964192:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077818.964311:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077818.964332:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077818/real 1769077818] req@ffff9b0365a39c00 x1855007617436544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077818.964354:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617436544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077818.964369:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077818.964371:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077818.964381:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077818.964386:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077818.964411:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077818.964429:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077818/real 1769077818] req@ffff9b0365a38a80 x1855007617436672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077818.964451:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617436672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077818.964456:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077818.964459:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077818.964461:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077819.475438:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077819.859354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077819.859354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077819.859411:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077820.499364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077820.883374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077820.883374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077820.883405:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077821.523382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077821.907330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077821.907363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077821.907402:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077822.547346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077822.931347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077822.931347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077822.931411:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077823.571349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077823.955351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077823.955351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077823.955384:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077824.083369:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077824.083380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077824.083385:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077824.083397:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4813 00000100:00080000:2.0:1769077824.083406:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077824.083414:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077824.083454:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617436800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077824.083490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077824.083494:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077824.083496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077824.083501:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4813 00000100:00080000:2.0:1769077824.083506:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077824.083510:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077824.083520:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077824.083543:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617436928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077824.083554:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617436800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077824.083561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077824.083567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077824.083570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077824.083586:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4813 00000100:00080000:2.0:1769077824.083592:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077824.083597:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077824.083605:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077824.083617:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077824.083622:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617437056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077824.083637:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077824.083641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077824.083643:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077824.083649:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4813 00000100:00080000:2.0:1769077824.083654:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077824.083657:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077824/real 1769077824] req@ffff9b03482c5880 x1855007617436800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077824.083663:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077824.083674:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077824.083684:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617436800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077824.083691:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617437184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077824.083705:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077824.083709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077824.083714:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:2.0:1769077824.083717:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769077824.083723:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617436800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077824.083723:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077824.083732:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4813 00000100:00080000:2.0:1769077824.083738:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077824.083742:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077824.083748:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077824.083754:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077824.083763:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617437312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077824.083785:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077824.083802:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077824/real 1769077824] req@ffff9b03482c5c00 x1855007617436928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077824.083812:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617436928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077824.083819:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077824.083821:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077824.083823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617436928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077824.083828:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077824.083843:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077824.083849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077824/real 1769077824] req@ffff9b03482c5f80 x1855007617437056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077824.083857:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617437056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077824.083863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077824.083865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077824.083882:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077824.083886:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077824.083901:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077824.083909:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077824/real 1769077824] req@ffff9b03482c7b80 x1855007617437184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077824.083917:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617437184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077824.083922:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077824.083924:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077824.083926:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077824.083939:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077824.083949:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077824.083955:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077824/real 1769077824] req@ffff9b03482c4e00 x1855007617437312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077824.083962:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617437312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077824.083971:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077824.083973:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077824.083975:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077824.595402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077824.979396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077824.979848:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077824.980571:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077825.619339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077826.003344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077826.003375:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077826.003986:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077826.643361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077827.027345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769077827.027602:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077827.027710:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077827.667346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077828.051789:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077828.052216:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077828.052899:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077828.691345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077829.075381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077829.075413:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077829.075632:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769077829.203315:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077829.203325:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077829.203330:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077829.203341:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4818 00000100:00080000:2.0:1769077829.203350:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077829.203357:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077829.203394:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617437440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077829.203425:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077829.203430:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077829.203432:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077829.203437:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4818 00000100:00080000:2.0:1769077829.203442:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077829.203446:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077829.203467:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077829.203475:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617437440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077829.203481:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617437568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077829.203491:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077829.203494:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077829.203496:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077829.203499:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4818 00000100:00080000:2.0:1769077829.203502:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077829.203505:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077829.203510:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077829.203523:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617437696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077829.203533:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077829.203537:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077829.203538:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077829.203542:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4818 00000800:00000100:1.0:1769077829.203547:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077829.203548:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077829.203556:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077829.203563:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077829.203577:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617437824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077829.203585:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077829/real 1769077829] req@ffff9b03482c4700 x1855007617437440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077829.203587:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077829.203595:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077829.203597:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077829.203601:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4818 00000100:00080000:2.0:1769077829.203604:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077829.203616:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077829.203620:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617437440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769077829.203624:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769077829.203633:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077829.203637:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077829.203640:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617437440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077829.203645:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617437952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077829.203664:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077829.203685:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077829.203693:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077829/real 1769077829] req@ffff9b03482c6a00 x1855007617437568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077829.203702:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617437568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077829.203708:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077829.203710:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077829.203713:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077829.203717:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077829.203727:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077829.203733:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077829/real 1769077829] req@ffff9b03482c4000 x1855007617437696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077829.203741:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617437696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077829.203746:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077829.203747:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077829.203749:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077829.203753:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077829.203771:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077829.203778:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077829/real 1769077829] req@ffff9b03482c7100 x1855007617437824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077829.203809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617437824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077829.203815:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077829.203817:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077829.203819:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077829.203823:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077829.203843:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077829.203851:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077829/real 1769077829] req@ffff9b03482c6680 x1855007617437952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077829.203858:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617437952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077829.203863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077829.203865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077829.203868:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617437952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077829.715373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077830.099432:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077830.099597:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077830.099603:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077830.739333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077831.123379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077831.123417:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077831.123444:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077831.763382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077832.147389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077832.147400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077832.147472:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077832.787411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077833.171403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077833.171409:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077833.171479:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077833.811354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077834.195370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077834.195376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077834.195390:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077834.323405:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077834.323421:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077834.323427:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077834.323440:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4823 00000100:00080000:3.0:1769077834.323453:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077834.323463:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077834.323510:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617438080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077834.323547:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077834.323553:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077834.323555:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077834.323558:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4823 00000100:00080000:3.0:1769077834.323562:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077834.323565:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077834.323575:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077834.323587:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617438208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077834.323595:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077834.323599:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077834.323601:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077834.323602:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617438080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077834.323605:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4823 00000100:00080000:3.0:1769077834.323608:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077834.323614:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077834.323620:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077834.323627:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617438336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077834.323634:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077834.323636:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077834.323637:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077834.323640:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4823 00000100:00080000:3.0:1769077834.323642:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077834.323645:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077834.323648:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077834.323656:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617438464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077834.323663:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077834.323666:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077834.323667:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077834.323670:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4823 00000100:00080000:3.0:1769077834.323673:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077834.323675:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077834.323679:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077834.323690:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617438592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077834.323716:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077834.323762:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077834/real 1769077834] req@ffff9b0352a18000 x1855007617438080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077834.323778:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617438080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077834.323788:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077834.323791:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077834.323795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617438080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077834.323819:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077834.323839:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077834.323847:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077834/real 1769077834] req@ffff9b0352a19500 x1855007617438208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077834.323856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617438208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077834.323863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077834.323865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077834.323867:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077834.323871:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077834.323882:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077834.323887:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077834/real 1769077834] req@ffff9b0352a1b800 x1855007617438336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077834.323894:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617438336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077834.323900:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077834.323902:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077834.323904:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077834.323907:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077834.323918:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077834.323923:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077834/real 1769077834] req@ffff9b0352a19c00 x1855007617438464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077834.323935:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617438464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077834.323941:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077834.323942:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077834.323944:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077834.323947:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077834.323956:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077834.323961:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077834/real 1769077834] req@ffff9b0352a18380 x1855007617438592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077834.323968:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617438592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077834.323973:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077834.323975:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077834.323977:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077834.835401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077835.219364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077835.219364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077835.219473:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077835.859328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077836.243343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077836.243343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077836.243377:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077836.883342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077837.267322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077837.267322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077837.267387:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077837.907332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077838.291351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077838.291351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077838.291413:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077838.931305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077839.315296:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077839.315312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077839.315344:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077839.443362:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077839.443371:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077839.443376:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077839.443387:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4828 00000100:00080000:3.0:1769077839.443397:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077839.443404:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077839.443454:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617438720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077839.443485:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077839.443488:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077839.443489:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077839.443503:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4828 00000100:00080000:3.0:1769077839.443506:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077839.443509:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077839.443516:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077839.443525:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617438848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077839.443533:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077839.443536:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077839.443537:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077839.443540:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4828 00000100:00080000:3.0:1769077839.443542:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077839.443544:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077839.443549:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077839.443557:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617438720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077839.443558:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617438976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077839.443571:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077839.443573:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077839.443575:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077839.443577:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4828 00000100:00080000:3.0:1769077839.443580:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077839.443582:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077839.443585:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077839.443594:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617439104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077839.443600:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077839.443603:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077839.443604:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077839.443606:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4828 00000100:00080000:3.0:1769077839.443641:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077839.443644:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077839.443648:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769077839.443649:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077839.443662:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617439232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077839.443689:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077839/real 1769077839] req@ffff9b0352a1ad80 x1855007617438720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077839.443704:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617438720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077839.443712:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077839.443715:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077839.443718:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617438720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077839.443724:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077839.443740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077839.443747:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077839/real 1769077839] req@ffff9b0352a1b100 x1855007617438848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077839.443755:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617438848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077839.443760:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077839.443762:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077839.443764:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077839.443782:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077839.443792:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077839.443797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077839/real 1769077839] req@ffff9b0352a18a80 x1855007617438976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077839.443811:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617438976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077839.443822:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077839.443824:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077839.443826:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617438976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077839.443830:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077839.443839:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077839.443844:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077839/real 1769077839] req@ffff9b0352a1a680 x1855007617439104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077839.443851:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617439104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077839.443872:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077839.443874:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077839.443876:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077839.443880:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077839.443891:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077839.443896:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077839/real 1769077839] req@ffff9b0352a1b480 x1855007617439232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077839.443904:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617439232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077839.443909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077839.443911:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077839.443913:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077839.955320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077840.339304:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077840.339319:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077840.339376:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077840.979360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077841.363312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077841.363312:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077841.363362:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077842.003349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077842.387339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077842.387396:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077842.387437:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077843.027364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077843.411325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077843.411328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077843.411455:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077844.051357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077844.435321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077844.435373:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077844.435378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769077844.563313:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077844.563326:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077844.563331:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077844.563342:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4833 00000100:00080000:3.0:1769077844.563354:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077844.563361:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077844.563404:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617439360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077844.563432:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077844.563436:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077844.563437:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077844.563441:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4833 00000100:00080000:3.0:1769077844.563444:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077844.563446:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077844.563456:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077844.563466:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617439488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077844.563475:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077844.563479:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077844.563481:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077844.563488:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4833 00000100:00100000:1.0:1769077844.563490:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617439360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077844.563492:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077844.563504:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077844.563509:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077844.563520:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617439616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077844.563543:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077844.563546:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077844.563548:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077844.563552:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4833 00000100:00080000:3.0:1769077844.563555:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077844.563559:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077844.563564:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077844.563572:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617439744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077844.563580:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077844.563583:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077844.563584:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077844.563587:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4833 00000100:00080000:3.0:1769077844.563589:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077844.563594:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077844.563599:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077844.563612:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617439872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077844.563631:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077844.563660:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077844.563682:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077844.563703:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077845.075310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077845.459330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077845.459356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077845.459422:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077846.099305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077846.483355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077846.483383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077846.483406:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077847.123279:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077847.507403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077847.507403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077847.507438:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769077847.635570:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769077847.635623:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769077847.635644:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077847.635706:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077847.635728:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077847.635740:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769077847.635750:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:3.0:1769077847.635829:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077844/real 1769077847] req@ffff9b0352a18700 x1855007617439488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077847.635858:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617439488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077847.635871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077847.635877:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077847.635881:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077847.635889:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077844/real 1769077847] req@ffff9b0352a19f80 x1855007617439616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077847.635903:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617439616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077847.635911:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077847.635915:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077847.635919:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077847.635928:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077844/real 1769077847] req@ffff9b0352a1a300 x1855007617439744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077847.635941:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617439744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077847.635948:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077847.635951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077847.635955:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077847.635962:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077844/real 1769077847] req@ffff9b0352a18e00 x1855007617439872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077847.635975:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617439872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077847.635984:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077847.635988:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077847.635991:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617439872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077847.635999:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077844/real 1769077847] req@ffff9b0352a1aa00 x1855007617439360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077847.636012:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617439360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077847.636018:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077847.636022:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077847.636026:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617439360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769077848.147319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077848.531385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077848.531449:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077848.531540:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077849.171323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077849.555418:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077849.555430:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077849.555535:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077849.683365:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077849.683378:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077849.683383:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077849.683396:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4838 00000100:00080000:0.0:1769077849.683408:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077849.683416:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077849.683474:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617440000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077849.683505:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077849.683509:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077849.683510:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077849.683514:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4838 00000100:00080000:0.0:1769077849.683517:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077849.683521:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077849.683533:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077849.683550:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617440000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077849.683552:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617440128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077849.683569:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077849.683573:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077849.683576:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077849.683581:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4838 00000100:00080000:0.0:1769077849.683587:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077849.683592:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077849.683614:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077849.683625:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617440256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077849.683633:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077849.683636:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077849.683638:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077849.683641:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4838 00000100:00080000:0.0:1769077849.683644:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077849.683649:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077849.683651:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769077849.683655:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077849.683665:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617440384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077849.683673:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077849.683677:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077849.683678:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077849.683681:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4838 00000100:00080000:0.0:1769077849.683683:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077849.683686:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077849.683690:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077849.683699:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077849/real 1769077849] req@ffff9b0365a38a80 x1855007617440000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077849.683702:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617440512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077849.683744:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617440000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077849.683776:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077849.683782:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077849.683788:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617440000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077849.683817:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077849.683846:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077849.683862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077849/real 1769077849] req@ffff9b0365a39c00 x1855007617440128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077849.683875:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617440128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077849.683885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077849.683889:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077849.683893:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077849.683899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077849.683922:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077849.683934:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077849/real 1769077849] req@ffff9b0365a38e00 x1855007617440256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077849.683948:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617440256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077849.683956:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077849.683959:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077849.683962:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077849.683969:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077849.683999:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077849.684012:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077849/real 1769077849] req@ffff9b0365a3b480 x1855007617440384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077849.684025:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617440384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077849.684044:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077849.684052:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077849.684056:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077849.684063:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077849.684083:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077849.684094:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077849/real 1769077849] req@ffff9b0365a3aa00 x1855007617440512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077849.684106:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617440512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077849.684115:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077849.684119:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077849.684121:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077850.195293:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077850.579399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077850.579416:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077850.579508:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077851.219299:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077851.603356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077851.603364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077851.603464:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077852.243300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077852.627346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077852.627346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077852.627425:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077853.267288:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077853.651365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077853.651365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077853.651459:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077854.291317:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077854.675333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077854.675333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077854.675407:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077854.803374:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077854.803383:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077854.803387:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077854.803395:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4843 00000100:00080000:0.0:1769077854.803405:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077854.803412:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077854.803441:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617440640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077854.803463:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077854.803466:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077854.803467:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077854.803470:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4843 00000100:00080000:0.0:1769077854.803473:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077854.803477:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077854.803485:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077854.803494:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617440640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077854.803497:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617440768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077854.803508:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077854.803511:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077854.803512:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077854.803515:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4843 00000100:00080000:0.0:1769077854.803519:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077854.803521:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077854.803526:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077854.803536:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617440896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077854.803543:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077854.803546:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077854.803548:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077854.803551:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4843 00000100:00080000:0.0:1769077854.803555:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077854.803559:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:3.0:1769077854.803561:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769077854.803564:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077854.803576:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617441024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077854.803583:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077854.803585:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077854.803586:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077854.803590:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4843 00000100:00080000:0.0:1769077854.803594:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077854.803596:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077854.803600:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077854.803600:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077854/real 1769077854] req@ffff9b0365a3b100 x1855007617440640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769077854.803611:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617441152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077854.803620:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617440640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077854.803630:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077854.803634:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077854.803637:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617440640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077854.803664:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077854.803680:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077854.803689:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077854/real 1769077854] req@ffff9b0365a39500 x1855007617440768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077854.803697:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617440768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077854.803703:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077854.803705:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077854.803707:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077854.803711:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077854.803779:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077854.803788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077854/real 1769077854] req@ffff9b0365a3bb80 x1855007617440896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077854.803796:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617440896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077854.803806:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077854.803808:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077854.803810:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617440896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077854.803814:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077854.803826:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077854.803831:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077854/real 1769077854] req@ffff9b0365a3ad80 x1855007617441024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077854.803838:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617441024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077854.803843:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077854.803845:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077854.803847:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077854.803851:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077854.803860:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077854.803865:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077854/real 1769077854] req@ffff9b0365a39180 x1855007617441152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077854.803872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617441152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077854.803877:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077854.803879:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077854.803881:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077855.315316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077855.699365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077855.699365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077855.699409:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077856.339401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077856.723336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077856.723336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077856.723400:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077857.363333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077857.747292:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077857.747316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077857.747358:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077858.387444:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077858.771341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077858.771372:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077858.771383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077859.411433:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077859.795409:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077859.795410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077859.795485:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077859.923366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077859.923375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077859.923379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077859.923388:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4849 00000100:00080000:3.0:1769077859.923397:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077859.923403:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077859.923436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617441280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077859.923465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077859.923468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077859.923469:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077859.923472:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4849 00000100:00080000:3.0:1769077859.923475:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077859.923477:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077859.923484:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077859.923493:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617441408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077859.923501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077859.923503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077859.923505:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077859.923508:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4849 00000100:00080000:3.0:1769077859.923511:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077859.923516:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617441280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077859.923516:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077859.923524:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077859.923534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617441536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077859.923541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077859.923543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077859.923544:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077859.923547:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4849 00000100:00080000:3.0:1769077859.923549:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077859.923552:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077859.923555:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077859.923562:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617441664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077859.923569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077859.923571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077859.923572:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077859.923574:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4849 00000100:00080000:3.0:1769077859.923578:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077859.923583:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077859.923587:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077859.923593:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077859.923607:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617441792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077859.923632:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077859/real 1769077859] req@ffff9b0352a1aa00 x1855007617441280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077859.923649:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617441280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077859.923657:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077859.923660:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077859.923663:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617441280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077859.923668:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077859.923686:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077859.923698:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077859/real 1769077859] req@ffff9b0352a18e00 x1855007617441408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077859.923711:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617441408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077859.923720:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077859.923724:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077859.923727:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077859.923749:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077859.923765:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077859.923773:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077859/real 1769077859] req@ffff9b0352a1a300 x1855007617441536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077859.923786:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617441536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077859.923791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077859.923793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077859.923795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077859.923799:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077859.923810:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077859.923816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077859/real 1769077859] req@ffff9b0352a19f80 x1855007617441664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077859.923823:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617441664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077859.923828:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077859.923830:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077859.923832:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077859.923835:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077859.923853:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077859.923860:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077859/real 1769077859] req@ffff9b0352a18700 x1855007617441792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077859.923867:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617441792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077859.923876:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077859.923879:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077859.923881:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617441792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077860.435380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077860.819358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077860.819366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077860.819472:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077861.459389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077861.843404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077861.843405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077861.843473:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077862.483389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077862.867309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077862.867314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077862.867383:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077863.507419:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077863.891358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077863.891367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077863.891460:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077864.531396:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077864.915323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077864.915349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077864.915373:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077865.043330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077865.043339:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077865.043343:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077865.043351:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4854 00000100:00080000:3.0:1769077865.043359:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077865.043365:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077865.043398:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617441920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077865.043421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077865.043424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077865.043425:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077865.043428:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4854 00000100:00080000:3.0:1769077865.043431:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077865.043433:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077865.043440:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077865.043449:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617442048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077865.043456:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077865.043460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077865.043461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077865.043464:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4854 00000100:00080000:3.0:1769077865.043466:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077865.043469:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077865.043472:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077865.043481:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617441920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077865.043481:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617442176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077865.043495:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077865.043497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077865.043498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077865.043501:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4854 00000100:00080000:3.0:1769077865.043504:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077865.043506:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077865.043510:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077865.043517:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617442304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077865.043523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077865.043525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077865.043527:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077865.043529:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4854 00000100:00080000:3.0:1769077865.043531:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077865.043533:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077865.043538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077865.043545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617442432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077865.043553:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077865.043599:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff9b0352a18700 x1855007617441920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077865.043615:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617441920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077865.043622:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077865.043626:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077865.043629:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617441920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077865.043634:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077865.043652:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077865.043661:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff9b0352a19f80 x1855007617442048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077865.043669:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617442048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077865.043675:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077865.043677:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077865.043680:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077865.043697:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077865.043708:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077865.043714:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff9b0352a1a300 x1855007617442176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077865.043722:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617442176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077865.043727:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077865.043729:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077865.043731:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077865.043735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077865.043745:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077865.043750:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff9b0352a18e00 x1855007617442304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077865.043757:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617442304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077865.043762:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077865.043764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077865.043766:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077865.043770:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077865.043790:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077865.043797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077865/real 1769077865] req@ffff9b0352a1aa00 x1855007617442432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077865.043805:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617442432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077865.043810:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077865.043812:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077865.043814:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077865.555400:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077865.939332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077865.939359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077865.939397:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077866.579321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077866.963366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077866.963371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077866.963459:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077867.603392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077867.987355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077867.987355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077867.987444:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077868.627334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077869.011375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077869.011386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077869.011426:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077869.651371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077870.035367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077870.035388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077870.035472:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077870.163375:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077870.163384:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077870.163389:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077870.163400:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4859 00000100:00080000:0.0:1769077870.163410:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077870.163418:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077870.163455:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617442560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077870.163480:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077870.163483:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077870.163484:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077870.163488:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4859 00000100:00080000:0.0:1769077870.163491:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077870.163493:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077870.163500:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077870.163510:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617442688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077870.163518:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077870.163521:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077870.163522:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077870.163525:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4859 00000100:00080000:0.0:1769077870.163527:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077870.163529:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077870.163532:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077870.163537:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617442560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077870.163543:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617442816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077870.163551:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077870.163554:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077870.163555:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077870.163558:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4859 00000100:00080000:0.0:1769077870.163561:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077870.163563:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077870.163566:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077870.163574:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617442944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077870.163581:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077870.163583:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077870.163584:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077870.163587:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4859 00000100:00080000:0.0:1769077870.163597:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077870.163602:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769077870.163604:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769077870.163607:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077870.163617:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617443072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077870.163649:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077870/real 1769077870] req@ffff9b0365a39180 x1855007617442560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077870.163669:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617442560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077870.163680:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077870.163685:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077870.163690:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617442560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077870.163698:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077870.163718:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077870.163728:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077870/real 1769077870] req@ffff9b0365a3ad80 x1855007617442688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077870.163741:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617442688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077870.163751:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077870.163754:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077870.163758:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077870.163780:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077870.163797:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077870.163807:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077870/real 1769077870] req@ffff9b0365a3bb80 x1855007617442816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077870.163818:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617442816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077870.163828:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077870.163831:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077870.163834:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077870.163840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077870.163856:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077870.163864:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077870/real 1769077870] req@ffff9b0365a39500 x1855007617442944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077870.163877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617442944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077870.163885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077870.163888:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077870.163892:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617442944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077870.163898:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077870.163912:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077870.163921:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077870/real 1769077870] req@ffff9b0365a3b100 x1855007617443072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077870.163933:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617443072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077870.163942:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077870.163945:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077870.163948:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077870.675331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077871.059361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077871.059361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077871.059485:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077871.699324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077872.083338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077872.083339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077872.083412:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077872.723346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077873.107337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077873.107337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077873.107385:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077873.747315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077874.131352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077874.131354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077874.131372:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077874.771318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077875.155358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077875.155363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769077875.155407:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077875.283376:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077875.283389:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077875.283394:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077875.283404:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4864 00000100:00080000:3.0:1769077875.283415:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077875.283423:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077875.283461:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617443200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077875.283500:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077875.283504:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077875.283506:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077875.283511:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4864 00000100:00080000:3.0:1769077875.283514:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077875.283519:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077875.283529:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077875.283546:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617443200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077875.283547:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617443328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077875.283574:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077875.283578:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077875.283580:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077875.283585:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4864 00000100:00080000:3.0:1769077875.283590:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077875.283595:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077875.283600:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077875.283611:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077875.283617:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617443456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077875.283628:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077875.283633:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077875.283635:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077875.283639:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4864 00000100:00080000:3.0:1769077875.283645:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077875.283649:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077875/real 1769077875] req@ffff9b0352a1b480 x1855007617443200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077875.283653:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077875.283671:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617443200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077875.283673:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769077875.283684:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077875.283688:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077875.283693:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617443584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077875.283695:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617443200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077875.283710:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077875.283714:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077875.283717:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077875.283720:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443328:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769077875.283725:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4864 00000100:00080000:3.0:1769077875.283731:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077875.283740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077875.283745:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077875.283751:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077875/real 1769077875] req@ffff9b0352a1a680 x1855007617443328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077875.283752:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077875.283766:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617443328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077875.283772:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617443712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077875.283775:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077875.283783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077875.283790:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077875.283801:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077875.283824:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077875.283836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077875/real 1769077875] req@ffff9b0352a18a80 x1855007617443456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077875.283856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617443456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077875.283862:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077875.283864:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077875.283876:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077875.283880:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077875.283896:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077875.283903:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077875/real 1769077875] req@ffff9b0352a1b100 x1855007617443584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077875.283912:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617443584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077875.283917:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077875.283919:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077875.283921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077875.283931:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077875.283940:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077875.283946:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077875/real 1769077875] req@ffff9b0352a1ad80 x1855007617443712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077875.283953:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617443712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077875.283962:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077875.283964:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077875.283966:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077875.795399:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077876.179344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077876.179344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077876.179402:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077876.819375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077877.203349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769077877.203366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077877.203406:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077877.843356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077878.227354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077878.227567:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077878.227614:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077878.867348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077879.251362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077879.251511:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077879.251539:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077879.891346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077880.275338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077880.275375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077880.275406:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077880.403340:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077880.403352:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077880.403357:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077880.403367:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4869 00000100:00080000:3.0:1769077880.403376:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077880.403384:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077880.403419:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617443840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077880.403446:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077880.403450:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077880.403452:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077880.403456:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4869 00000100:00080000:3.0:1769077880.403459:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077880.403463:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077880.403472:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077880.403485:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617443968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077880.403493:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617443840:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077880.403500:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077880.403504:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077880.403506:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077880.403511:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4869 00000100:00080000:3.0:1769077880.403515:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077880.403518:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077880.403523:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077880.403533:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617444096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077880.403542:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077880.403545:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077880.403547:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769077880.403552:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077880.403553:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4869 00000100:00080000:3.0:1769077880.403559:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077880.403563:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077880.403570:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077880.403580:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617444224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077880.403589:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077880.403593:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077880.403594:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077880.403599:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4869 00000100:00100000:1.0:1769077880.403602:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077880/real 1769077880] req@ffff9b0352a18380 x1855007617443840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077880.403604:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077880.403611:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077880.403618:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077880.403627:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617443840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077880.403633:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617444352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077880.403641:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077880.403649:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077880.403654:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617443840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077880.403676:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077880.403694:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077880.403704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077880/real 1769077880] req@ffff9b0352a19c00 x1855007617443968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077880.403721:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617443968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077880.403729:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077880.403732:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077880.403735:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617443968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077880.403752:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077880.403767:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077880.403776:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077880/real 1769077880] req@ffff9b0352a1b800 x1855007617444096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077880.403787:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617444096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077880.403795:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077880.403797:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077880.403801:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077880.403806:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077880.403819:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077880.403826:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077880/real 1769077880] req@ffff9b0352a19500 x1855007617444224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077880.403836:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617444224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077880.403849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077880.403859:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077880.403862:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077880.403867:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077880.403880:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077880.403888:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077880/real 1769077880] req@ffff9b0352a18000 x1855007617444352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077880.403898:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617444352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077880.403906:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077880.403909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077880.403912:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077880.915296:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077881.299348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077881.299367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077881.299409:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077881.939324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077882.323292:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077882.323301:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077882.323556:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077882.963311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077883.347352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077883.347352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077883.347418:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077883.987360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077884.371398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077884.371441:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077884.371449:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077885.011388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077885.395364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077885.395364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077885.395444:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077885.523319:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077885.523334:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077885.523339:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077885.523351:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4874 00000100:00080000:3.0:1769077885.523363:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077885.523371:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077885.523416:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617444480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077885.523454:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077885.523458:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077885.523461:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077885.523466:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4874 00000100:00080000:3.0:1769077885.523471:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077885.523476:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077885.523486:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077885.523495:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617444480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077885.523502:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617444608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077885.523516:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077885.523519:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077885.523521:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077885.523526:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4874 00000100:00080000:3.0:1769077885.523529:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077885.523533:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077885.523539:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077885.523561:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617444736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077885.523572:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077885.523575:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077885.523577:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077885.523581:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4874 00000100:00080000:3.0:1769077885.523593:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077885.523598:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077885.523603:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077885.523614:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617444864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077885.523624:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077885.523627:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077885.523629:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077885.523634:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4874 00000100:00080000:3.0:1769077885.523637:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077885.523641:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077885.523646:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077885.523656:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617444992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077885.523665:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077885.523731:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077885/real 1769077885] req@ffff9b0352a18000 x1855007617444480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077885.523752:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617444480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077885.523764:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077885.523769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077885.523773:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617444480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077885.523800:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077885.523825:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077885.523855:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077885/real 1769077885] req@ffff9b0352a19500 x1855007617444608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077885.523868:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617444608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077885.523877:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077885.523881:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077885.523884:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077885.523891:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077885.523912:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077885.523923:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077885/real 1769077885] req@ffff9b0352a1b800 x1855007617444736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077885.523935:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617444736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077885.523942:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077885.523946:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077885.523949:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077885.523955:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077885.523982:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077885.523994:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077885/real 1769077885] req@ffff9b0352a19c00 x1855007617444864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077885.524006:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617444864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077885.524014:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077885.524017:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077885.524020:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077885.524025:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077885.524041:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077885.524050:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077885/real 1769077885] req@ffff9b0352a18380 x1855007617444992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077885.524061:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617444992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077885.524069:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077885.524072:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077885.524075:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617444992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077886.035356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077886.419368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077886.419368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077886.419443:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077887.059354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077887.443348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077887.443348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077887.443415:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077888.083363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077888.467357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077888.467440:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077888.467493:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077889.107425:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077889.491341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077889.491341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077889.491428:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077890.131360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077890.515327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077890.515327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077890.515389:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077890.643366:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077890.643374:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077890.643378:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077890.643386:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4879 00000100:00080000:0.0:1769077890.643394:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077890.643400:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077890.643429:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617445120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077890.643453:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077890.643456:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077890.643457:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077890.643461:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4879 00000100:00080000:0.0:1769077890.643464:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077890.643466:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077890.643474:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077890.643483:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617445248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077890.643490:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077890.643493:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077890.643494:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077890.643498:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4879 00000100:00100000:3.0:1769077890.643500:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617445120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077890.643501:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077890.643506:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077890.643511:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077890.643519:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617445376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077890.643524:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077890.643527:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077890.643528:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077890.643531:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4879 00000100:00080000:0.0:1769077890.643534:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077890.643536:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077890.643540:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077890.643547:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617445504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077890.643553:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077890.643555:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077890.643557:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077890.643558:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4879 00000100:00080000:0.0:1769077890.643560:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077890.643562:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077890.643565:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769077890.643568:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769077890.643573:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617445632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077890.643682:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077890/real 1769077890] req@ffff9b0365a3aa00 x1855007617445120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077890.643809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617445120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077890.643819:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077890.643824:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077890.643875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617445120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077890.643894:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617445248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077890.643915:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077890.643928:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077890/real 1769077890] req@ffff9b0365a3b480 x1855007617445248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077890.643938:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617445248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077890.643946:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077890.643949:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077890.643952:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617445248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077890.643957:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617445376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077890.643974:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077890.643983:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077890/real 1769077890] req@ffff9b0365a38e00 x1855007617445376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077890.643993:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617445376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077890.643999:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077890.644002:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077890.644005:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617445376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077890.644011:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617445504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077890.644027:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077890.644036:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077890/real 1769077890] req@ffff9b0365a39c00 x1855007617445504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077890.644046:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617445504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077890.644052:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077890.644054:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077890.644057:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617445504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077890.644063:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617445632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077890.644078:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077890.644088:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077890/real 1769077890] req@ffff9b0365a38a80 x1855007617445632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077890.644098:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617445632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077890.644105:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077890.644108:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077890.644111:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617445632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077891.155345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077891.539692:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077891.539692:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077891.540369:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077892.179300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077892.563472:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077892.563552:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077892.563659:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077893.203332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077893.587352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077893.587525:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077893.587617:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077894.227295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077894.611695:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077894.611711:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077894.611724:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077895.251346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769077895.635869:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077895.636086:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077895.636215:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769077895.763367:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077895.763378:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077895.763383:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077895.763393:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4884 00000100:00080000:3.0:1769077895.763404:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077895.763410:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077895.763456:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617445760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077895.763500:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077895.763506:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077895.763508:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077895.763512:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4884 00000100:00080000:3.0:1769077895.763515:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077895.763519:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077895.763528:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077895.763540:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617445888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077895.763550:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077895.763555:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617445760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077895.763555:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077895.763558:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077895.763566:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4884 00000100:00080000:3.0:1769077895.763569:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077895.763571:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077895.763575:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077895.763585:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617446016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077895.763591:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077895.763601:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077895.763602:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077895.763605:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4884 00000100:00080000:3.0:1769077895.763608:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077895.763611:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077895.763615:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077895.763623:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617446144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077895.763630:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077895.763632:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077895.763633:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077895.763637:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4884 00000100:00080000:3.0:1769077895.763640:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077895.763642:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077895.763647:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077895.763652:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077895.763694:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617446272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077895.763722:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077895/real 1769077895] req@ffff9b0352a1ad80 x1855007617445760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077895.763740:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617445760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077895.763751:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077895.763756:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077895.763760:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617445760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077895.763788:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617445888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077895.763813:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077895.763824:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077895/real 1769077895] req@ffff9b0352a1b100 x1855007617445888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077895.763833:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617445888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077895.763839:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077895.763842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077895.763844:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617445888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077895.763850:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077895.763862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077895.763868:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077895/real 1769077895] req@ffff9b0352a18a80 x1855007617446016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077895.763876:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617446016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077895.763881:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077895.763883:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077895.763885:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077895.763889:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077895.763900:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077895.763905:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077895/real 1769077895] req@ffff9b0352a1a680 x1855007617446144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077895.763912:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617446144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077895.763918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077895.763919:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077895.763921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077895.763924:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077895.763933:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077895.763938:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077895/real 1769077895] req@ffff9b0352a1b480 x1855007617446272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077895.763946:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617446272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077895.763951:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077895.763953:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077895.763955:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077896.275319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769077896.659479:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077896.659494:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769077896.659582:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077897.299332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077897.683386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077897.683387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077897.683510:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077898.323366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077898.707325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077898.707346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077898.707422:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077899.347367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077899.731309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077899.731336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769077899.731391:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077900.371398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077900.755351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077900.755363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077900.755444:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077900.883318:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077900.883330:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077900.883334:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077900.883345:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4890 00000100:00080000:3.0:1769077900.883353:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077900.883359:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077900.883391:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617446400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077900.883416:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077900.883421:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077900.883423:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077900.883427:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4890 00000100:00080000:3.0:1769077900.883432:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077900.883437:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077900.883448:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077900.883458:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617446400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077900.883465:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617446528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077900.883477:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077900.883479:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077900.883481:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077900.883484:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4890 00000100:00080000:3.0:1769077900.883487:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077900.883490:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077900.883494:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077900.883503:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617446656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077900.883510:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077900.883512:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077900.883513:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077900.883516:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4890 00000800:00000100:1.0:1769077900.883520:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077900.883520:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077900.883526:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077900.883531:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077900.883540:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617446784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077900.883547:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077900.883549:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077900.883551:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077900.883554:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4890 00000100:00080000:3.0:1769077900.883556:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077900.883558:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077900.883560:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077900.883569:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077900/real 1769077900] req@ffff9b0352a1aa00 x1855007617446400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077900.883571:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617446912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077900.883620:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617446400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077900.883628:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077900.883632:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077900.883635:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617446400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077900.883653:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077900.883671:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077900.883680:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077900/real 1769077900] req@ffff9b0352a18e00 x1855007617446528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077900.883697:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617446528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077900.883706:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077900.883709:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077900.883712:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077900.883719:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077900.883739:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077900.883751:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077900/real 1769077900] req@ffff9b0352a1a300 x1855007617446656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077900.883761:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617446656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077900.883767:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077900.883769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077900.883771:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077900.883775:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077900.883789:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077900.883796:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077900/real 1769077900] req@ffff9b0352a19f80 x1855007617446784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077900.883803:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617446784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077900.883813:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077900.883815:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077900.883817:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077900.883821:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077900.883830:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077900.883835:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077900/real 1769077900] req@ffff9b0352a18700 x1855007617446912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077900.883842:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617446912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077900.883847:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077900.883849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077900.883851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617446912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077901.395343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077901.779306:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077901.779306:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077901.779458:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077902.419387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077902.803373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077902.803401:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077902.803442:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077903.443359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077903.827347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077903.827449:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077903.827484:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077904.467357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077904.851370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077904.851370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769077904.851436:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077905.491362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077905.875357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077905.875397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077905.875454:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077906.003357:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077906.003369:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077906.003373:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077906.003385:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4895 00000100:00080000:3.0:1769077906.003395:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077906.003402:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077906.003447:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617447040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077906.003470:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077906.003475:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077906.003477:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077906.003481:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4895 00000100:00080000:3.0:1769077906.003486:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077906.003520:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077906.003524:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617447040:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769077906.003529:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077906.003543:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617447168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077906.003553:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077906.003556:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077906.003558:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077906.003562:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4895 00000100:00080000:3.0:1769077906.003565:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077906.003568:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077906.003572:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077906.003581:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617447296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077906.003584:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077906.003590:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077906.003593:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077906.003595:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077906.003598:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4895 00000100:00080000:3.0:1769077906.003601:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077906.003605:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077906.003610:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077906.003618:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617447424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077906.003622:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9b0352a18700 x1855007617447040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077906.003628:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077906.003631:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077906.003633:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077906.003636:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4895 00000100:00100000:1.0:1769077906.003639:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617447040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077906.003640:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077906.003643:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077906.003648:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769077906.003657:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769077906.003657:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617447552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077906.003662:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077906.003666:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617447040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077906.003689:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077906.003713:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077906.003724:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9b0352a19f80 x1855007617447168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077906.003733:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617447168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077906.003739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077906.003741:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077906.003744:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077906.003748:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077906.003760:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077906.003766:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9b0352a1a300 x1855007617447296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077906.003773:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617447296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077906.003778:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077906.003780:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077906.003781:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077906.003794:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077906.003806:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077906.003813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9b0352a18e00 x1855007617447424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077906.003820:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617447424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077906.003825:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077906.003827:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077906.003829:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077906.003832:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077906.003842:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077906.003847:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077906/real 1769077906] req@ffff9b0352a1aa00 x1855007617447552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077906.003854:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617447552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077906.003859:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077906.003861:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077906.003862:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077906.515348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077906.899337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077906.899338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077906.899542:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077907.539349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077907.923315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077907.923317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077907.923379:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077908.563381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077908.947314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077908.947326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077908.947414:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077909.587376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077909.971366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077909.971369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077909.971444:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077910.611367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077910.995366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077910.995366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077910.995452:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077911.123373:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077911.123388:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077911.123393:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077911.123420:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4900 00000100:00080000:3.0:1769077911.123433:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077911.123442:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077911.123489:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617447680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077911.123518:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077911.123522:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077911.123523:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077911.123527:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4900 00000100:00080000:3.0:1769077911.123529:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077911.123532:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077911.123541:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077911.123550:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617447808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077911.123558:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077911.123561:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077911.123562:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077911.123565:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4900 00000100:00080000:3.0:1769077911.123567:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077911.123570:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077911.123573:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077911.123582:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617447680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077911.123583:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617447936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077911.123598:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077911.123600:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077911.123601:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077911.123605:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4900 00000100:00080000:3.0:1769077911.123607:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077911.123609:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077911.123614:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077911.123623:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617448064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077911.123630:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077911.123633:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077911.123634:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077911.123636:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4900 00000100:00080000:3.0:1769077911.123638:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077911.123641:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077911.123643:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077911.123650:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617448192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077911.123720:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077911.123773:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077911.123810:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077911.123840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077911.635307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077912.019400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077912.019423:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077912.019480:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077912.659301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077913.043353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077913.043353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077913.043410:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077913.683310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077914.067342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077914.067342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077914.067411:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1769077914.195547:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:3.0:1769077914.195588:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769077914.195610:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769077914.195639:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769077914.195653:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769077914.195661:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769077914.195670:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769077914.195672:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077911/real 1769077914] req@ffff9b0352a18a80 x1855007617447936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077914.195699:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617447936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077914.195712:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077914.195718:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077914.195723:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077914.195733:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077911/real 1769077914] req@ffff9b0352a1b100 x1855007617448064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077914.195747:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617448064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077914.195755:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077914.195759:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077914.195762:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077914.195770:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077911/real 1769077914] req@ffff9b0352a1ad80 x1855007617448192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077914.195780:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617448192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077914.195786:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077914.195789:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077914.195791:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077914.195797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077911/real 1769077914] req@ffff9b0352a1b480 x1855007617447680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077914.195809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617447680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077914.195816:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077914.195820:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077914.195823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617447680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077914.195830:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077911/real 1769077914] req@ffff9b0352a1a680 x1855007617447808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077914.195844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617447808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077914.195854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077914.195859:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077914.195862:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617447808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077914.707332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077915.091353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077915.091353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077915.091439:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077915.731319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077916.115361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077916.115369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077916.115434:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077916.243392:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077916.243404:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077916.243410:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077916.243421:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4905 00000100:00080000:1.0:1769077916.243433:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077916.243441:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077916.243482:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617448320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077916.243514:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077916.243518:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077916.243519:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077916.243523:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4905 00000100:00080000:1.0:1769077916.243526:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077916.243528:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077916.243537:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077916.243547:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617448448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077916.243557:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077916.243559:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077916.243561:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077916.243568:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4905 00000100:00080000:1.0:1769077916.243572:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077916.243576:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077916.243576:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617448320:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769077916.243581:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077916.243595:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617448576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077916.243602:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077916.243604:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077916.243606:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077916.243609:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4905 00000100:00080000:1.0:1769077916.243613:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077916.243617:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077916.243620:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077916.243632:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617448704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077916.243638:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077916.243641:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077916.243643:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077916.243647:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4905 00000100:00080000:1.0:1769077916.243649:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077916.243652:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077916.243657:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077916.243668:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617448832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769077916.243681:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077916.243735:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077916/real 1769077916] req@ffff9b0343deca80 x1855007617448320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077916.243758:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617448320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077916.243771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077916.243776:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077916.243781:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617448320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077916.243808:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077916.243832:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077916.243845:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077916/real 1769077916] req@ffff9b0343deed80 x1855007617448448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077916.243859:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617448448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077916.243868:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077916.243872:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077916.243876:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077916.243883:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077916.243902:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077916.243911:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077916/real 1769077916] req@ffff9b0343ded500 x1855007617448576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077916.243931:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617448576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077916.243939:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077916.243955:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077916.243959:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077916.243966:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077916.243988:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077916.243999:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077916/real 1769077916] req@ffff9b0343def480 x1855007617448704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077916.244012:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617448704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077916.244020:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077916.244023:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077916.244026:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077916.244032:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077916.244049:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077916.244059:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077916/real 1769077916] req@ffff9b0343dee680 x1855007617448832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077916.244071:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617448832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077916.244085:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077916.244089:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077916.244092:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617448832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769077916.755358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077917.139365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077917.139375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077917.139427:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077917.779361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077918.163366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077918.163370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077918.163397:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077918.803427:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077919.187362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077919.187362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077919.187425:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077919.827335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077920.211365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077920.211433:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077920.211478:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077920.851344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077921.235302:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077921.235316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077921.235375:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077921.363375:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077921.363384:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077921.363388:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077921.363396:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4910 00000100:00080000:3.0:1769077921.363404:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077921.363409:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077921.363442:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617448960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077921.363465:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077921.363468:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077921.363469:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077921.363473:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4910 00000100:00080000:3.0:1769077921.363475:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077921.363485:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077921.363495:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077921.363514:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617448960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077921.363514:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617449088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077921.363533:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077921.363538:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077921.363540:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077921.363553:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4910 00000100:00080000:3.0:1769077921.363558:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077921.363563:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077921.363569:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077921.363584:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617449216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077921.363592:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077921.363602:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077921.363607:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077921.363608:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077921.363612:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4910 00000100:00080000:3.0:1769077921.363617:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077921.363620:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077921.363624:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077921.363636:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617449344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077921.363644:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077921/real 1769077921] req@ffff9b0352a18380 x1855007617448960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077921.363646:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077921.363651:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077921.363653:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077921.363658:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4910 00000100:00080000:3.0:1769077921.363665:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077921.363670:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077921.363674:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077921.363683:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617448960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077921.363689:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617449472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077921.363718:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077921.363727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077921.363733:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617448960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077921.363760:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077921.363786:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077921.363800:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077921/real 1769077921] req@ffff9b0352a19c00 x1855007617449088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077921.363825:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617449088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077921.363835:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077921.363839:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077921.363843:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077921.363850:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077921.363872:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077921.363884:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077921/real 1769077921] req@ffff9b0352a1b800 x1855007617449216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077921.363897:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617449216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077921.363906:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077921.363909:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077921.363913:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077921.363921:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077921.363943:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077921.363955:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077921/real 1769077921] req@ffff9b0352a19500 x1855007617449344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077921.363968:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617449344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077921.363977:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077921.363980:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077921.363983:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077921.363990:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077921.364012:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077921.364023:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077921/real 1769077921] req@ffff9b0352a18000 x1855007617449472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077921.364036:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617449472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077921.364045:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077921.364048:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077921.364052:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077921.875373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077922.259338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077922.259338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077922.259408:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077922.899348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077923.283291:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077923.283298:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077923.283385:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077923.923324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077924.307352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077924.307352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077924.307434:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077924.947404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077925.331371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077925.331393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077925.331417:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077925.971390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077926.355412:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077926.355413:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077926.355509:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077926.483374:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077926.483385:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077926.483391:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077926.483401:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4915 00000100:00080000:3.0:1769077926.483413:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077926.483422:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077926.483463:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617449600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077926.483497:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077926.483501:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077926.483503:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077926.483508:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4915 00000100:00080000:3.0:1769077926.483512:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077926.483516:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077926.483526:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077926.483547:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617449600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077926.483555:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617449728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077926.483569:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077926.483573:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077926.483575:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077926.483580:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4915 00000100:00080000:3.0:1769077926.483585:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077926.483589:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077926.483595:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077926.483606:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077926.483610:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617449856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077926.483623:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077926.483626:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077926.483628:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077926.483634:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4915 00000100:00080000:3.0:1769077926.483640:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077926.483644:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077926.483650:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077926.483656:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077926/real 1769077926] req@ffff9b0352a18000 x1855007617449600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077926.483667:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617449984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077926.483673:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617449600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077926.483693:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077926.483694:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077926.483701:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769077926.483702:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1769077926.483707:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617449600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077926.483707:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077926.483717:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4915 00000100:00080000:3.0:1769077926.483723:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077926.483727:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077926.483733:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449728:192.168.204.160@tcp:38:kworker.0 00000008:00100000:3.0:1769077926.483734:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077926.483753:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617450112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077926.483761:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077926.483775:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077926/real 1769077926] req@ffff9b0352a19500 x1855007617449728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077926.483785:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617449728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077926.483791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077926.483794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077926.483796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077926.483802:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077926.483815:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077926.483821:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077926/real 1769077926] req@ffff9b0352a1b800 x1855007617449856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077926.483830:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617449856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077926.483836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077926.483838:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077926.483841:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077926.483846:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077926.483868:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077926.483882:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077926/real 1769077926] req@ffff9b0352a19c00 x1855007617449984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077926.483898:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617449984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077926.483908:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077926.483912:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077926.483916:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617449984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077926.483932:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617450112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077926.483954:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077926.483964:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077926/real 1769077926] req@ffff9b0352a18380 x1855007617450112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077926.483983:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617450112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077926.483990:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077926.483993:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077926.483995:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617450112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077926.995350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077927.379354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077927.379354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077927.379424:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077928.019353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077928.403301:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077928.403302:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077928.403405:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077929.043336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077929.427336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077929.427336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077929.427408:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077930.067350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077930.451370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077930.451378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077930.451451:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077931.091352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077931.475338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077931.475338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077931.475415:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077931.603383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077931.603394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077931.603399:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077931.603409:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4920 00000100:00080000:0.0:1769077931.603419:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077931.603426:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077931.603463:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617450240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077931.603494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077931.603498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077931.603500:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077931.603505:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4920 00000100:00080000:0.0:1769077931.603509:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077931.603513:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077931.603522:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077931.603538:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617450368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077931.603548:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617450240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077931.603555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077931.603560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077931.603563:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077931.603568:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4920 00000100:00080000:0.0:1769077931.603573:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077931.603577:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077931.603583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077931.603596:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617450496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077931.603607:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077931.603611:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077931.603613:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769077931.603617:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077931.603622:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4920 00000100:00080000:0.0:1769077931.603628:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077931.603634:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077931.603640:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077931.603652:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617450624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077931.603663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077931.603668:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077931/real 1769077931] req@ffff9b0365a38a80 x1855007617450240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077931.603669:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077931.603673:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077931.603683:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4920 00000100:00080000:0.0:1769077931.603688:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077931.603695:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077931.603695:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617450240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1769077931.603705:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769077931.603709:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077931.603715:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769077931.603720:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617450752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077931.603721:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617450240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077931.603746:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617450368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077931.603765:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077931.603774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077931/real 1769077931] req@ffff9b0365a39c00 x1855007617450368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077931.603782:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617450368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077931.603788:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077931.603790:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077931.603792:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617450368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077931.603796:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617450496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077931.603806:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077931.603812:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077931/real 1769077931] req@ffff9b0365a38e00 x1855007617450496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077931.603819:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617450496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077931.603827:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077931.603830:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077931.603834:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617450496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077931.603840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617450624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077931.603862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077931.603873:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077931/real 1769077931] req@ffff9b0365a3b480 x1855007617450624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077931.603886:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617450624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077931.603895:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077931.603898:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077931.603902:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617450624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077931.603908:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617450752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077931.603925:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077931.603932:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077931/real 1769077931] req@ffff9b0365a3aa00 x1855007617450752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077931.603940:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617450752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077931.603948:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077931.603951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077931.603954:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617450752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077932.115342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077932.499342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077932.499342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077932.499388:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077933.139322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077933.523325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077933.523325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077933.523379:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077934.163351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077934.547338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077934.547338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077934.547427:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077935.187293:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077935.571339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077935.571339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077935.571409:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077936.211352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077936.595341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077936.595341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077936.595408:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077936.723369:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077936.723378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077936.723382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077936.723391:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4925 00000100:00080000:3.0:1769077936.723399:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077936.723405:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077936.723438:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617450880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077936.723464:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077936.723468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077936.723470:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077936.723474:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4925 00000100:00080000:3.0:1769077936.723479:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077936.723483:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077936.723493:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077936.723507:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617451008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077936.723517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769077936.723519:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617450880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077936.723521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077936.723523:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077936.723526:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4925 00000100:00080000:3.0:1769077936.723530:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077936.723534:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077936.723538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077936.723547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617451136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077936.723556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077936.723559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077936.723561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077936.723564:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4925 00000100:00080000:3.0:1769077936.723567:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077936.723570:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077936.723575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769077936.723581:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077936.723583:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617451264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077936.723597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077936.723600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077936.723602:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077936.723605:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4925 00000100:00080000:3.0:1769077936.723608:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077936.723611:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077936.723616:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077936.723618:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077936/real 1769077936] req@ffff9b0352a1a680 x1855007617450880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077936.723627:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617451392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077936.723635:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617450880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077936.723645:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077936.723649:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077936.723654:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617450880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077936.723679:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077936.723701:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077936.723714:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077936/real 1769077936] req@ffff9b0352a1b480 x1855007617451008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077936.723729:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617451008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077936.723738:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077936.723740:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077936.723743:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077936.723747:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077936.723763:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077936.723780:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077936/real 1769077936] req@ffff9b0352a1ad80 x1855007617451136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077936.723789:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617451136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077936.723795:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077936.723797:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077936.723799:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077936.723803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077936.723815:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077936.723822:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077936/real 1769077936] req@ffff9b0352a1b100 x1855007617451264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077936.723829:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617451264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077936.723834:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077936.723836:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077936.723837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077936.723841:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077936.723850:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077936.723855:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077936/real 1769077936] req@ffff9b0352a18a80 x1855007617451392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077936.723862:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617451392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077936.723867:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077936.723869:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077936.723872:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077937.235354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077937.619348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077937.619348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077937.619411:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077938.259352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077938.643308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077938.643310:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077938.643363:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077939.283340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077939.667338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077939.667338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077939.667395:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077940.307325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077940.691293:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077940.691311:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077940.691993:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077941.331344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077941.715339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077941.715339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077941.715493:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077941.843315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077941.843325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077941.843329:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077941.843338:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4931 00000100:00080000:3.0:1769077941.843346:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077941.843352:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077941.843384:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617451520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077941.843403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077941.843406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077941.843408:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077941.843411:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4931 00000100:00080000:3.0:1769077941.843414:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077941.843417:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077941.843424:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077941.843433:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617451520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077941.843436:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617451648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077941.843446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077941.843448:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077941.843450:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077941.843452:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4931 00000100:00080000:3.0:1769077941.843455:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077941.843457:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077941.843468:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077941.843480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617451776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077941.843487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077941.843489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077941.843490:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077941.843493:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4931 00000100:00080000:3.0:1769077941.843496:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769077941.843501:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077941.843502:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077941.843508:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077941.843517:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617451904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077941.843524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077941.843526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077941.843528:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077941.843530:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4931 00000100:00080000:3.0:1769077941.843533:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077941.843535:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077941.843539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077941.843546:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077941/real 1769077941] req@ffff9b0352a1aa00 x1855007617451520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077941.843550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617452032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077941.843588:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617451520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077941.843601:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077941.843606:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077941.843611:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617451520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077941.843631:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077941.843650:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077941.843660:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077941/real 1769077941] req@ffff9b0352a18e00 x1855007617451648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077941.843673:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617451648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077941.843679:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077941.843682:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077941.843684:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077941.843688:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077941.843698:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077941.843704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077941/real 1769077941] req@ffff9b0352a1a300 x1855007617451776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077941.843711:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617451776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077941.843716:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077941.843718:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077941.843720:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077941.843723:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077941.843733:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077941.843738:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077941/real 1769077941] req@ffff9b0352a19f80 x1855007617451904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077941.843745:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617451904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077941.843755:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077941.843757:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077941.843759:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617451904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077941.843762:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077941.843771:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077941.843776:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077941/real 1769077941] req@ffff9b0352a18700 x1855007617452032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077941.843783:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617452032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077941.843788:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077941.843790:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077941.843792:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077942.355824:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077942.739327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077942.739327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077942.739391:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077943.379353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077943.763347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077943.763417:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077943.763473:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077944.403327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077944.787353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077944.787353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077944.787412:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077945.427361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077945.811351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077945.811406:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077945.811447:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077946.451349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077946.835374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077946.835374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077946.835441:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077946.963335:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077946.963347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077946.963351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077946.963362:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4936 00000100:00080000:3.0:1769077946.963373:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077946.963380:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077946.963442:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617452160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077946.963475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077946.963479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077946.963481:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077946.963484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4936 00000100:00080000:3.0:1769077946.963488:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077946.963491:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077946.963501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077946.963514:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617452288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077946.963523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077946.963528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1769077946.963529:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617452160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077946.963530:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077946.963538:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4936 00000100:00080000:3.0:1769077946.963541:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077946.963544:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077946.963547:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077946.963556:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617452416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077946.963562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077946.963565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077946.963566:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077946.963570:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4936 00000100:00080000:3.0:1769077946.963572:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077946.963574:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077946.963579:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077946.963586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617452544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077946.963593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077946.963595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077946.963596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077946.963598:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4936 00000100:00080000:3.0:1769077946.963602:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077946.963604:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077946.963607:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769077946.963618:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077946.963624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617452672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077946.963665:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077946/real 1769077946] req@ffff9b0352a18700 x1855007617452160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077946.963682:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617452160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077946.963692:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077946.963695:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077946.963698:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617452160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077946.963722:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077946.963738:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077946.963755:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077946/real 1769077946] req@ffff9b0352a19f80 x1855007617452288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077946.963765:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617452288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077946.963771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077946.963774:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077946.963776:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077946.963780:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077946.963792:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077946.963799:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077946/real 1769077946] req@ffff9b0352a1a300 x1855007617452416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077946.963828:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617452416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077946.963844:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077946.963849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077946.963852:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077946.963860:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077946.963884:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077946.963897:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077946/real 1769077946] req@ffff9b0352a18e00 x1855007617452544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077946.963919:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617452544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077946.963928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077946.963932:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077946.963936:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077946.963943:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077946.963963:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077946.963973:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077946/real 1769077946] req@ffff9b0352a1aa00 x1855007617452672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077946.963986:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617452672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077946.963994:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077946.963997:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077946.964001:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077947.475382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077947.859399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077947.859399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077947.859488:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077948.499341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077948.883370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077948.883382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077948.883483:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077949.523376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077949.907387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077949.907388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077949.907483:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077950.547296:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077950.931317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077950.931318:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077950.931444:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077951.571371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077951.955361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077951.955361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077951.955460:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077952.083373:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077952.083386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077952.083391:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077952.083402:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4941 00000100:00080000:3.0:1769077952.083416:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077952.083425:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077952.083460:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617452800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077952.083494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077952.083498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077952.083500:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077952.083504:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4941 00000100:00080000:3.0:1769077952.083509:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077952.083513:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077952.083524:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077952.083539:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617452928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077952.083549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077952.083552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077952.083553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077952.083559:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4941 00000100:00080000:3.0:1769077952.083566:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1769077952.083569:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617452800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077952.083573:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077952.083585:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077952.083595:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617453056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077952.083601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077952.083604:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077952.083605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077952.083609:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4941 00000100:00080000:3.0:1769077952.083613:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077952.083615:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077952.083618:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077952.083625:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617453184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077952.083632:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077952.083634:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077952.083636:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077952.083638:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4941 00000100:00080000:3.0:1769077952.083642:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077952.083645:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077952.083647:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077952.083653:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617453312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077952.083679:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077952.083767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077952/real 1769077952] req@ffff9b0352a18a80 x1855007617452800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077952.083790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617452800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077952.083802:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077952.083808:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077952.083813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617452800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077952.083843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077952.083870:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077952.083884:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077952/real 1769077952] req@ffff9b0352a1b100 x1855007617452928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077952.083898:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617452928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077952.083909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077952.083913:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077952.083917:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617452928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077952.083924:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077952.083945:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077952.083956:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077952/real 1769077952] req@ffff9b0352a1ad80 x1855007617453056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077952.083970:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617453056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077952.083978:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077952.083981:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077952.083997:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077952.084007:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077952.084027:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077952.084037:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077952/real 1769077952] req@ffff9b0352a1b480 x1855007617453184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077952.084048:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617453184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077952.084056:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077952.084059:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077952.084063:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077952.084069:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077952.084084:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077952.084093:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077952/real 1769077952] req@ffff9b0352a1a680 x1855007617453312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077952.084104:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617453312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077977 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077952.084119:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077952.084123:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077952.084127:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077952.595314:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077952.979368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077952.979368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077952.979449:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077953.619364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077954.003355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077954.003355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077954.003405:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077954.643386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077955.027402:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077955.027407:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077955.027535:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077955.667401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077956.051411:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077956.051412:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077956.051484:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077956.691378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077957.075365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077957.075370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077957.075450:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077957.203375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077957.203387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077957.203391:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077957.203403:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4946 00000100:00080000:3.0:1769077957.203415:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077957.203424:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077957.203463:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617453440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077957.203492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077957.203496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077957.203498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077957.203504:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4946 00000100:00080000:3.0:1769077957.203508:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077957.203513:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077957.203522:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617453440:192.168.204.160@tcp:250:kworker.0 00000008:00100000:3.0:1769077957.203523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077957.203542:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617453568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077957.203554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077957.203558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077957.203560:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077957.203564:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4946 00000100:00080000:3.0:1769077957.203568:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077957.203572:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769077957.203584:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769077957.203589:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077957.203607:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617453696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077957.203617:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077957.203620:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077957.203622:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077957.203641:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4946 00000100:00100000:1.0:1769077957.203642:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077957/real 1769077957] req@ffff9b0352a18380 x1855007617453440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077957.203647:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077957.203656:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077957.203665:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077957.203671:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617453440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077957.203679:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617453824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077957.203688:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077957.203693:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077957.203698:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769077957.203701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077957.203704:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769077957.203706:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617453440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077957.203711:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4946 00000100:00080000:3.0:1769077957.203717:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077957.203720:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077957.203725:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077957.203737:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077957.203739:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617453952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077957.203764:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077957.203776:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077957/real 1769077957] req@ffff9b0352a19c00 x1855007617453568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077957.203796:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617453568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077957.203803:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077957.203805:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077957.203807:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077957.203812:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077957.203824:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077957.203830:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077957/real 1769077957] req@ffff9b0352a1b800 x1855007617453696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077957.203838:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617453696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077957.203843:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077957.203845:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077957.203847:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077957.203851:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077957.203861:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077957.203866:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077957/real 1769077957] req@ffff9b0352a19500 x1855007617453824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077957.203873:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617453824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077957.203878:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077957.203879:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077957.203882:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077957.203892:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077957.203902:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077957.203907:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077957/real 1769077957] req@ffff9b0352a18000 x1855007617453952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077957.203914:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617453952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077957.203919:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077957.203921:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077957.203930:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617453952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077957.715369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769077958.099880:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077958.100133:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769077958.100294:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077958.739364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077959.123421:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077959.123499:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077959.123516:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077959.763361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077960.147375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077960.147785:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077960.147878:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077960.787379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077961.171546:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077961.171849:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769077961.171928:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077961.811344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077962.195892:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077962.195894:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769077962.195937:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077962.323382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077962.323392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077962.323396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077962.323404:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4951 00000100:00080000:0.0:1769077962.323412:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077962.323419:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077962.323449:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617454080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077962.323479:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077962.323482:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077962.323484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077962.323487:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4951 00000100:00080000:0.0:1769077962.323490:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077962.323492:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077962.323499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077962.323510:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617454208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077962.323517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077962.323520:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077962.323522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077962.323526:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4951 00000100:00080000:0.0:1769077962.323531:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1769077962.323532:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617454080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077962.323540:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077962.323575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077962.323591:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617454336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077962.323603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077962.323607:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077962.323610:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769077962.323614:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769077962.323619:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4951 00000100:00080000:0.0:1769077962.323625:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077962.323629:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077962.323634:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077962.323646:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617454464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077962.323654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1769077962.323655:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077962/real 1769077962] req@ffff9b0365a3b100 x1855007617454080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769077962.323660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077962.323663:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077962.323666:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4951 00000100:00080000:0.0:1769077962.323669:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077962.323671:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077962.323674:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077962.323680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617454592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077962.323733:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617454080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077962.323743:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077962.323746:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077962.323749:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617454080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077962.323846:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077962.323868:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077962.323918:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077962/real 1769077962] req@ffff9b0365a39500 x1855007617454208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077962.323933:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617454208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077962.323939:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077962.323942:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077962.323945:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077962.323965:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077962.323979:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077962.323986:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077962/real 1769077962] req@ffff9b0365a3bb80 x1855007617454336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077962.323994:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617454336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077962.323999:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077962.324001:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077962.324003:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077962.324007:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077962.324017:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077962.324023:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077962/real 1769077962] req@ffff9b0365a3ad80 x1855007617454464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077962.324030:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617454464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077962.324036:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077962.324037:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077962.324039:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077962.324042:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077962.324052:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077962.324057:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077962/real 1769077962] req@ffff9b0365a39180 x1855007617454592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077962.324064:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617454592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077962.324068:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077962.324070:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077962.324072:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077962.835394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077963.219811:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077963.219825:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077963.219918:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077963.859330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077964.243380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077964.243392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077964.243438:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077964.883341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077965.267335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077965.267335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077965.267394:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077965.907353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077966.291347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077966.291347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077966.291443:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077966.931363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077967.315342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077967.315342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077967.315447:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077967.443351:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077967.443362:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077967.443367:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077967.443378:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4956 00000100:00080000:3.0:1769077967.443385:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077967.443392:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077967.443425:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617454720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077967.443453:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077967.443486:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077967.443489:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077967.443495:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4956 00000100:00080000:3.0:1769077967.443500:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077967.443506:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077967.443516:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077967.443518:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617454720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077967.443534:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617454848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077967.443546:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077967.443550:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077967.443553:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077967.443557:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4956 00000100:00080000:3.0:1769077967.443562:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077967.443566:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077967.443572:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1769077967.443587:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077967.443588:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617454976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077967.443600:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077967.443605:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077967.443607:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077967.443612:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4956 00000100:00080000:3.0:1769077967.443616:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077967.443621:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077967.443625:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077967/real 1769077967] req@ffff9b0352a18000 x1855007617454720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769077967.443628:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077967.443644:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617455104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077967.443653:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617454720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077967.443660:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077967.443666:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077967.443666:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769077967.443669:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077967.443673:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769077967.443677:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4956 00000100:00100000:1.0:1769077967.443680:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617454720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769077967.443682:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077967.443689:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077967.443694:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077967.443704:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077967.443706:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617455232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077967.443724:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077967.443735:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077967/real 1769077967] req@ffff9b0352a19500 x1855007617454848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077967.443744:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617454848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077967.443750:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077967.443752:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077967.443754:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077967.443758:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077967.443768:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077967.443774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077967/real 1769077967] req@ffff9b0352a1b800 x1855007617454976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077967.443785:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617454976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077967.443799:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077967.443803:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077967.443806:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617454976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077967.443812:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077967.443833:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077967.443844:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077967/real 1769077967] req@ffff9b0352a19c00 x1855007617455104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077967.443857:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617455104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077967.443865:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077967.443867:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077967.443869:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077967.443881:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077967.443895:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077967.443902:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077967/real 1769077967] req@ffff9b0352a18380 x1855007617455232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077967.443909:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617455232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077967.443915:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077967.443916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077967.443919:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077967.955340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077968.339349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077968.339349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077968.339392:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077968.979350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077969.363354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077969.363354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077969.363438:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077970.003349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077970.387347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077970.387347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769077970.387393:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077971.027387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077971.411319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077971.411345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077971.411456:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077972.051337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077972.435362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077972.435389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077972.435440:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769077972.563347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077972.563363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769077972.563397:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077972.563411:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4961 00000100:00080000:3.0:1769077972.563424:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077972.563434:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769077972.563485:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617455360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077972.563508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077972.563512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769077972.563513:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077972.563520:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4961 00000100:00080000:3.0:1769077972.563523:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077972.563527:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077972.563537:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077972.563549:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617455488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077972.563558:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077972.563560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769077972.563562:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077972.563565:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4961 00000100:00080000:3.0:1769077972.563566:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077972.563570:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077972.563576:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077972.563583:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617455360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077972.563589:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617455616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077972.563602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077972.563605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769077972.563607:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077972.563613:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4961 00000100:00080000:3.0:1769077972.563617:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077972.563621:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077972.563627:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769077972.563641:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617455744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769077972.563652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769077972.563657:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1769077972.563660:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769077972.563663:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769077972.563671:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4961 00000100:00080000:3.0:1769077972.563678:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769077972.563684:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769077972.563691:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077972.563701:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077972/real 1769077972] req@ffff9b0352a1a680 x1855007617455360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077972.563710:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617455872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077972.563725:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617455360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077972.563738:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077972.563742:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077972.563747:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617455360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077972.563755:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077972.563770:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077972.563778:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077972/real 1769077972] req@ffff9b0352a1b480 x1855007617455488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077972.563786:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617455488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077972.563791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077972.563793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077972.563796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077972.563816:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077972.563827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077972.563833:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077972/real 1769077972] req@ffff9b0352a1ad80 x1855007617455616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077972.563841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617455616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077972.563846:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077972.563848:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077972.563850:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077972.563854:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077972.563864:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077972.563869:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077972/real 1769077972] req@ffff9b0352a1b100 x1855007617455744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077972.563877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617455744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077972.563882:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077972.563883:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077972.563885:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077972.563889:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077972.563897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077972.563903:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077972/real 1769077972] req@ffff9b0352a18a80 x1855007617455872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077972.563911:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617455872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769077997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077972.563916:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077972.563918:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077972.563920:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617455872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077973.075336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077973.459308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077973.459308:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077973.459486:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077974.099337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077974.483351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077974.483354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077974.483413:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077975.123360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077975.507341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769077975.507443:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077975.507467:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077976.147360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077976.531376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769077976.531382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077976.531483:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077977.171359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077977.555309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077977.555325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077977.555380:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769077977.683401:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077977.683414:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769077977.683419:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077977.683430:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4966 00000100:00080000:0.0:1769077977.683440:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077977.683447:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769077977.683485:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617456000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077977.683513:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077977.683517:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769077977.683519:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077977.683524:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4966 00000100:00080000:0.0:1769077977.683529:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077977.683533:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077977.683543:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077977.683556:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617456128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077977.683561:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617456000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769077977.683567:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077977.683571:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769077977.683573:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077977.683576:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4966 00000100:00080000:0.0:1769077977.683579:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077977.683583:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077977.683587:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769077977.683595:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617456256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077977.683603:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077977.683606:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769077977.683608:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077977.683612:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4966 00000100:00080000:0.0:1769077977.683615:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077977.683619:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077977.683623:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769077977.683631:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617456384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769077977.683639:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769077977.683642:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769077977.683644:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769077977.683647:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4966 00000100:00080000:0.0:1769077977.683651:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769077977.683655:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769077977.683660:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077977.683662:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769077977.683669:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617456512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077977.683694:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077977.683719:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077977.683742:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077978.195318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077978.579355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077978.579358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077978.579433:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077979.219383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077979.603324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077979.603388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077979.603468:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077980.243316:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077980.627363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077980.627363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077980.627432:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769077980.755478:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769077980.755521:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769077980.755546:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077980.755581:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077980.755590:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077980.755598:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769077980.755608:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769077980.755640:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077977/real 1769077980] req@ffff9b0365a3b100 x1855007617456512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077980.755668:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617456512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077980.755682:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077980.755688:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077980.755693:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077980.755701:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077977/real 1769077980] req@ffff9b0365a3ad80 x1855007617456128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077980.755714:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617456128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077980.755721:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077980.755724:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077980.755728:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077980.755734:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077977/real 1769077980] req@ffff9b0365a3bb80 x1855007617456256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077980.755747:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617456256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077980.755756:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077980.755759:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077980.755762:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077980.755768:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077977/real 1769077980] req@ffff9b0365a39500 x1855007617456384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077980.755780:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617456384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077980.755788:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077980.755792:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077980.755795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077980.755801:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077977/real 1769077980] req@ffff9b0365a39180 x1855007617456000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077980.755814:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617456000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077980.755820:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077980.755824:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077980.755827:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617456000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769077981.267346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077981.651353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077981.651362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077981.651472:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077982.291545:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077982.675378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077982.675378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077982.675530:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769077982.803415:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077982.803424:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077982.803427:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077982.803435:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4971 00000100:00080000:2.0:1769077982.803445:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077982.803451:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077982.803482:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617456640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077982.803521:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077982.803524:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077982.803526:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077982.803531:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4971 00000100:00080000:2.0:1769077982.803535:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077982.803539:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077982.803549:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077982.803563:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617456768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077982.803575:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617456640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077982.803595:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077982.803600:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077982.803602:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077982.803605:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4971 00000100:00080000:2.0:1769077982.803609:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077982.803613:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077982.803617:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077982.803626:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617456896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769077982.803646:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769077982.803650:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077982.803654:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077982.803657:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077982.803662:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4971 00000100:00080000:2.0:1769077982.803667:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077982.803671:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077982.803677:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077982.803690:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617457024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077982.803696:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077982/real 1769077982] req@ffff9b03482c6d80 x1855007617456640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077982.803703:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077982.803710:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077982.803713:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077982.803718:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4971 00000100:00080000:2.0:1769077982.803721:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077982.803724:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077982.803729:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617456640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1769077982.803729:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077982.803743:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617457152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077982.803744:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077982.803753:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077982.803760:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617456640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077982.803803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077982.803853:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077982.803866:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077982/real 1769077982] req@ffff9b03482c6300 x1855007617456768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077982.803878:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617456768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077982.803887:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077982.803891:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077982.803894:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077982.803900:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077982.803919:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077982.803936:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077982/real 1769077982] req@ffff9b03482c5500 x1855007617456896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077982.803949:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617456896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077982.803957:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077982.803960:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077982.803964:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617456896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077982.803970:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077982.803990:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077982.804001:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077982/real 1769077982] req@ffff9b03482c4a80 x1855007617457024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077982.804013:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617457024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077982.804021:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077982.804024:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077982.804027:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077982.804032:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077982.804050:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077982.804060:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077982/real 1769077982] req@ffff9b03482c5180 x1855007617457152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077982.804070:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617457152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077982.804077:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077982.804080:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077982.804083:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077983.316091:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077983.699347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077983.699347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077983.699976:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077984.340061:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077984.723335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077984.723335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077984.723391:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077985.363606:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077985.747339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077985.747343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077985.747410:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077986.388020:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077986.771345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077986.771346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077986.771408:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077987.411385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077987.795398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077987.795582:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769077987.800171:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769077987.923373:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077987.923383:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769077987.923396:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077987.923409:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4977 00000100:00080000:2.0:1769077987.923418:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077987.923425:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077987.923462:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617457280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077987.923489:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077987.923493:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769077987.923494:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077987.923498:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4977 00000100:00080000:2.0:1769077987.923502:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077987.923508:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077987.923518:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077987.923532:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617457408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077987.923547:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617457280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769077987.923550:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077987.923564:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769077987.923567:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077987.923572:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4977 00000100:00080000:2.0:1769077987.923577:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077987.923583:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077987.923588:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769077987.923602:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617457536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077987.923612:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077987.923616:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769077987.923618:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077987.923622:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4977 00000100:00080000:2.0:1769077987.923626:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077987.923630:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077987.923636:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769077987.923650:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077987.923651:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617457664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769077987.923664:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769077987.923669:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769077987.923671:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769077987.923678:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4977 00000100:00080000:2.0:1769077987.923683:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769077987.923689:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769077987.923696:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077987.923705:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077987/real 1769077987] req@ffff9b03482c4380 x1855007617457280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077987.923713:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617457792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769077987.923728:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617457280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077987.923757:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077987.923764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077987.923775:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617457280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077987.923803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077987.923821:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077987.923829:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077987/real 1769077987] req@ffff9b03482c5180 x1855007617457408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077987.923836:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617457408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077987.923842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077987.923844:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077987.923846:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077987.923849:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769077987.923860:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077987.923865:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077987/real 1769077987] req@ffff9b03482c4a80 x1855007617457536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077987.923872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617457536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077987.923880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077987.923883:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077987.923885:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769077987.923888:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077987.923897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077987.923902:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077987/real 1769077987] req@ffff9b03482c5500 x1855007617457664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077987.923908:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617457664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077987.923912:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077987.923914:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077987.923916:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769077987.923919:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769077987.923927:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077987.923932:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077987/real 1769077987] req@ffff9b03482c6300 x1855007617457792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077987.923939:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617457792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077987.923943:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077987.923945:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769077987.923947:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617457792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077988.435385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769077988.819364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077988.819364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077988.819469:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077989.459352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077989.843385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077989.843423:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077989.843447:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077990.483373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077990.867380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077990.867393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077990.867445:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077991.507312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077991.891357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077991.891367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077991.891424:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077992.531350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077992.915356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077992.915423:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077992.915458:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:1.0:1769077993.043346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077993.043357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077993.043361:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077993.043370:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4982 00000100:00080000:1.0:1769077993.043378:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077993.043383:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077993.043419:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617457920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077993.043440:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077993.043443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077993.043445:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077993.043449:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4982 00000100:00080000:1.0:1769077993.043452:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077993.043456:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077993.043465:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769077993.043469:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617457920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077993.043475:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617458048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077993.043484:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077993.043487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077993.043489:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077993.043492:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4982 00000100:00080000:1.0:1769077993.043495:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077993.043497:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077993.043501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077993.043509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617458176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077993.043517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077993.043519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077993.043521:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1769077993.043523:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769077993.043524:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4982 00000100:00080000:1.0:1769077993.043527:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077993.043530:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077993.043534:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077993.043541:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617458304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077993.043548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077993.043551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077993.043552:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077993.043555:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4982 00000100:00100000:3.0:1769077993.043556:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff9b0343dece00 x1855007617457920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077993.043558:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077993.043561:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077993.043565:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769077993.043572:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617458432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769077993.043590:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617457920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077993.043603:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077993.043608:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077993.043611:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617457920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769077993.043631:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077993.043649:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077993.043657:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff9b0343defb80 x1855007617458048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077993.043665:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617458048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077993.043670:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077993.043672:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077993.043674:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077993.043677:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769077993.043686:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077993.043691:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff9b0343dee680 x1855007617458176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077993.043697:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617458176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077993.043708:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077993.043710:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077993.043712:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769077993.043715:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077993.043725:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077993.043731:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff9b0343def480 x1855007617458304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077993.043741:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617458304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077993.043746:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077993.043748:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077993.043749:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769077993.043752:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769077993.043760:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769077993.043764:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077993/real 1769077993] req@ffff9b0343ded500 x1855007617458432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769077993.043770:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617458432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769077993.043774:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769077993.043776:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769077993.043778:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077993.555353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077993.939380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077993.939380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077993.939543:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077994.579351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077994.963326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077994.963326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077994.963389:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077995.603362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077995.987334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077995.987404:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077995.987429:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769077996.627334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077997.011305:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077997.011308:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077997.011505:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077997.651339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769077998.035334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769077998.035335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769077998.035406:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769077998.163332:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077998.163342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769077998.163347:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077998.163357:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4987 00000100:00080000:1.0:1769077998.163368:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077998.163374:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769077998.163404:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617458560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077998.163437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077998.163441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769077998.163445:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077998.163450:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4987 00000100:00080000:1.0:1769077998.163454:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077998.163466:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769077998.163475:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617458560:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769077998.163480:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769077998.163498:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617458688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077998.163511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077998.163514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769077998.163517:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077998.163521:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4987 00000100:00080000:1.0:1769077998.163526:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077998.163531:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077998.163536:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769077998.163542:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769077998.163555:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617458816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077998.163568:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769077998.163571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769077998.163573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077998.163577:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4987 00000100:00080000:1.0:1769077998.163581:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077998.163585:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769077998.163592:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077998.163602:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077998/real 1769077998] req@ffff9b0343deed80 x1855007617458560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769077998.163605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617458944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769077998.163623:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769077998.163627:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617458560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769077998.163628:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769077998.163632:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769077998.163641:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4987 00000100:00080000:2.0:1769077998.163644:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769077998.163646:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769077998.163653:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769077998.163655:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000008:00100000:1.0:1769077998.163661:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769077998.163665:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617458560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769077998.163676:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617459072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769077998.163699:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077998.163720:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077998.163732:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077998/real 1769077998] req@ffff9b0343deca80 x1855007617458688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077998.163743:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617458688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077998.163750:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077998.163755:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077998.163758:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077998.163764:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769077998.163782:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077998.163792:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077998/real 1769077998] req@ffff9b0343deea00 x1855007617458816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077998.163803:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617458816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077998.163809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077998.163811:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077998.163813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769077998.163816:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077998.163829:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077998.163835:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077998/real 1769077998] req@ffff9b0343def800 x1855007617458944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077998.163842:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617458944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077998.163849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077998.163851:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077998.163853:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617458944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077998.163856:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769077998.163866:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769077998.163878:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769077998/real 1769077998] req@ffff9b0343ded880 x1855007617459072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769077998.163887:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617459072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769077998.163893:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769077998.163895:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769077998.163898:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769077998.675337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769077999.059317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769077999.059329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769077999.059380:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769077999.699326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078000.083358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078000.083358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078000.083413:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078000.723374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078001.107299:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078001.107302:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078001.107351:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078001.747330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078002.131337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078002.131337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078002.131423:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078002.771366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078003.155360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078003.155364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078003.155430:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078003.283374:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078003.283390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078003.283396:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078003.283410:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4992 00000100:00080000:1.0:1769078003.283422:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078003.283581:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078003.283622:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617459200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078003.283649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078003.283652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078003.283655:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078003.283659:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4992 00000100:00080000:1.0:1769078003.283666:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078003.283670:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078003.283678:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078003.283687:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617459328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078003.283696:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617459200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078003.283710:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078003.283713:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078003.283714:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078003.283717:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4992 00000100:00080000:1.0:1769078003.283719:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078003.283722:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078003.283725:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078003.283735:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617459456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078003.283741:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078003.283743:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078003.283745:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078003.283748:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4992 00000100:00080000:1.0:1769078003.283751:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078003.283753:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078003.283757:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078003.283760:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078003.283769:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617459584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078003.283777:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078003.283780:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078003.283781:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078003.283784:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4992 00000100:00080000:1.0:1769078003.283787:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078003.283790:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078003.283794:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078003.283799:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078003/real 1769078003] req@ffff9b0343dedf80 x1855007617459200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078003.283804:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617459712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078003.283826:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617459200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078003.283842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078003.283845:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078003.283848:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617459200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078003.283891:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078003.283933:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078003.283943:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078003/real 1769078003] req@ffff9b0343def100 x1855007617459328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078003.283952:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617459328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078003.283958:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078003.283960:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078003.283963:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078003.283967:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078003.283978:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078003.283994:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078003/real 1769078003] req@ffff9b0343dee300 x1855007617459456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078003.284002:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617459456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078003.284007:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078003.284009:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078003.284011:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078003.284015:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078003.284026:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078003.284031:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078003/real 1769078003] req@ffff9b0343dec380 x1855007617459584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078003.284039:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617459584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078003.284043:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078003.284045:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078003.284047:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078003.284050:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078003.284059:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078003.284064:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078003/real 1769078003] req@ffff9b0343ded180 x1855007617459712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078003.284070:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617459712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078003.284076:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078003.284078:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078003.284081:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078003.795338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078004.179338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078004.179338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078004.179372:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078004.819353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078005.203324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078005.203324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078005.203349:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078005.843353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078006.227346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078006.227348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078006.227425:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078006.867356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078007.251344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078007.251345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078007.251431:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078007.891344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078008.275350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078008.275350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078008.275418:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078008.403378:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078008.403386:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078008.403390:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078008.403397:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 4997 00000100:00080000:2.0:1769078008.403405:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078008.403410:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078008.403443:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617459840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078008.403468:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078008.403472:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078008.403474:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078008.403479:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4997 00000100:00080000:2.0:1769078008.403484:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078008.403489:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078008.403498:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617459840:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769078008.403499:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078008.403519:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617459968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078008.403530:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078008.403534:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078008.403535:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078008.403538:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4997 00000100:00080000:2.0:1769078008.403541:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078008.403544:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078008.403547:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078008.403555:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617460096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078008.403561:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078008.403563:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078008.403564:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078008.403567:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4997 00000100:00080000:2.0:1769078008.403568:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078008.403570:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078008.403573:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078008.403578:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617460224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078008.403588:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078008.403590:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078008.403592:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078008.403594:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 4997 00000100:00080000:2.0:1769078008.403595:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078008.403597:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078008.403599:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078008.403604:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617460352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769078008.405582:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078008.405630:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078008/real 1769078008] req@ffff9b03482c6d80 x1855007617459840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078008.405645:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617459840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078008.405653:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078008.405656:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078008.405659:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617459840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078008.405691:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078008.405712:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078008.405721:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078008/real 1769078008] req@ffff9b03482c6680 x1855007617459968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078008.405730:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617459968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078008.405736:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078008.405739:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078008.405741:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617459968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078008.405747:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078008.405781:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078008.405789:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078008/real 1769078008] req@ffff9b03482c7100 x1855007617460096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078008.405797:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617460096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078008.405802:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078008.405804:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078008.405807:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078008.405810:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078008.405822:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078008.405828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078008/real 1769078008] req@ffff9b03482c4000 x1855007617460224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078008.405835:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617460224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078008.405840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078008.405842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078008.405844:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078008.405847:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078008.405856:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078008.405862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078008/real 1769078008] req@ffff9b03482c6a00 x1855007617460352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078008.405870:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617460352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078008.405875:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078008.405877:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078008.405879:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078008.915310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078009.299347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078009.299358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078009.299429:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078009.939341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078010.323360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078010.323369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078010.323380:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078010.963315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078011.347348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078011.347348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078011.347665:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078011.987350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078012.371361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078012.371365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078012.371450:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078013.011342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078013.395350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078013.395363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078013.395414:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078013.523372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078013.523382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078013.523394:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078013.523403:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5002 00000100:00080000:3.0:1769078013.523417:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078013.523423:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078013.523475:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617460480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078013.523499:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078013.523503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078013.523504:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078013.523508:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5002 00000100:00080000:3.0:1769078013.523512:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078013.523516:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078013.523525:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078013.523534:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617460608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078013.523545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078013.523552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769078013.523555:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617460480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078013.523555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078013.523568:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5002 00000100:00080000:3.0:1769078013.523572:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078013.523575:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078013.523578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078013.523592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617460736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078013.523599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078013.523601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078013.523603:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078013.523606:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5002 00000100:00080000:3.0:1769078013.523608:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078013.523611:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078013.523615:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078013.523624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617460864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078013.523630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078013.523633:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078013.523636:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078013.523639:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5002 00000100:00080000:3.0:1769078013.523643:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078013.523645:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769078013.523651:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769078013.523653:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078013.523667:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617460992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078013.523725:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078013/real 1769078013] req@ffff9b0352a1aa00 x1855007617460480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078013.523745:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617460480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078013.523760:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078013.523764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078013.523767:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617460480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078013.523795:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078013.523814:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078013.523823:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078013/real 1769078013] req@ffff9b0352a18e00 x1855007617460608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078013.523832:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617460608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078013.523839:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078013.523842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078013.523844:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078013.523848:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078013.523859:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078013.523864:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078013/real 1769078013] req@ffff9b0352a1a300 x1855007617460736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078013.523872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617460736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078013.523878:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078013.523880:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078013.523882:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078013.523926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078013.523941:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078013.523949:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078013/real 1769078013] req@ffff9b0352a19f80 x1855007617460864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078013.523957:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617460864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078013.523963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078013.523966:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078013.523968:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078013.523972:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078013.523986:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078013.523996:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078013/real 1769078013] req@ffff9b0352a18700 x1855007617460992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078013.524018:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617460992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078013.524028:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078013.524032:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078013.524036:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617460992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078014.035381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078014.419334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078014.419364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078014.419463:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078015.059386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078015.443361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078015.443370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078015.443398:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078016.083343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078016.467355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078016.467355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078016.467424:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078017.107384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078017.491364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078017.491381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078017.491444:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078018.131354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078018.515344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078018.515377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078018.515416:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078018.643421:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078018.643430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078018.643434:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078018.643442:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5007 00000100:00080000:1.0:1769078018.643452:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078018.643458:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078018.643496:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617461120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078018.643527:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078018.643531:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078018.643533:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078018.643537:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5007 00000100:00080000:1.0:1769078018.643541:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078018.643545:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078018.643553:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078018.643566:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617461248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078018.643576:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617461120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078018.643582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078018.643588:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078018.643590:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078018.643593:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5007 00000100:00080000:1.0:1769078018.643611:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078018.643615:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078018.643619:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078018.643632:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617461376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078018.643638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078018.643641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078018.643642:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078018.643645:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5007 00000100:00080000:1.0:1769078018.643649:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078018.643652:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078018.643658:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078018.643666:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617461504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078018.643672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078018.643676:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078018.643677:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078018.643679:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5007 00000100:00080000:1.0:1769078018.643685:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078018.643687:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078018.643690:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078018.643704:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078018.643705:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617461632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078018.643774:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078018/real 1769078018] req@ffff9b0343dec700 x1855007617461120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078018.643798:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617461120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078018.643807:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078018.643812:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078018.643815:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617461120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078018.643843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617461248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078018.643864:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078018.643873:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078018/real 1769078018] req@ffff9b0343dec000 x1855007617461248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078018.643882:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617461248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078018.643888:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078018.643891:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078018.643893:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617461248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078018.643899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617461376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078018.643912:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078018.643918:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078018/real 1769078018] req@ffff9b0343ded180 x1855007617461376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078018.643927:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617461376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078018.643932:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078018.643934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078018.643936:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617461376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078018.643940:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617461504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078018.643951:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078018.643957:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078018/real 1769078018] req@ffff9b0343dec380 x1855007617461504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078018.643964:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617461504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078018.643969:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078018.643971:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078018.643973:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617461504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078018.643977:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617461632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078018.643991:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078018.644001:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078018/real 1769078018] req@ffff9b0343dee300 x1855007617461632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078018.644015:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617461632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078018.644023:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078018.644027:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078018.644030:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617461632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078019.155362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078019.539341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078019.539352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078019.539419:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078020.179369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078020.563400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078020.563401:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078020.563444:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078021.203339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078021.587334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078021.587341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078021.587411:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078022.227351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078022.611355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078022.611355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078022.611423:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078023.251477:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078023.635348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078023.635348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078023.635417:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078023.763362:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078023.763370:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078023.763374:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078023.763382:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5012 00000100:00080000:3.0:1769078023.763391:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078023.763396:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078023.763431:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617461760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078023.763451:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078023.763454:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078023.763455:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078023.763460:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5012 00000100:00080000:3.0:1769078023.763462:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078023.763466:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078023.763474:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078023.763484:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617461760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078023.763492:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617461888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078023.763501:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078023.763504:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078023.763505:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078023.763520:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5012 00000100:00080000:3.0:1769078023.763524:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078023.763528:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078023.763533:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078023.763543:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617462016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078023.763550:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078023.763553:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078023.763554:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078023.763557:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5012 00000100:00080000:3.0:1769078023.763560:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078023.763562:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078023.763566:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078023.763570:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078023.763577:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617462144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078023.763586:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078023.763588:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078023.763590:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078023.763593:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5012 00000100:00080000:3.0:1769078023.763595:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078023.763597:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078023.763600:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078023.763608:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617462272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078023.763620:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078023/real 1769078023] req@ffff9b0352a18700 x1855007617461760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078023.763638:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617461760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078023.763646:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078023.763649:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078023.763652:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617461760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078023.763671:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617461888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078023.763694:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078023.763706:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078023/real 1769078023] req@ffff9b0352a19f80 x1855007617461888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078023.763720:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617461888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078023.763729:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078023.763732:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078023.763736:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617461888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078023.763742:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078023.763764:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078023.763775:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078023/real 1769078023] req@ffff9b0352a1a300 x1855007617462016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078023.763788:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617462016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078023.763796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078023.763800:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078023.763804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078023.763810:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078023.763832:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078023.763843:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078023/real 1769078023] req@ffff9b0352a18e00 x1855007617462144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078023.763855:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617462144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078023.763864:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078023.763867:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078023.763871:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078023.763877:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078023.763897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078023.763908:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078023/real 1769078023] req@ffff9b0352a1aa00 x1855007617462272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078023.763920:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617462272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078023.763928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078023.763932:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078023.763935:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078024.275335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078024.659483:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078024.659558:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078024.660960:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078025.299345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078025.684294:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078025.686555:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078025.686644:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078026.323331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769078026.707544:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078026.707659:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078026.708336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078027.347372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078027.731432:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078027.731908:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078027.731939:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078028.371348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078028.755385:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078028.755986:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078028.756093:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078028.883375:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078028.883387:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078028.883392:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078028.883405:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5018 00000100:00080000:3.0:1769078028.883416:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078028.883439:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078028.883489:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617462400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078028.883517:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078028.883521:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078028.883523:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078028.883526:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5018 00000100:00080000:3.0:1769078028.883529:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078028.883534:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078028.883544:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078028.883554:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617462528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078028.883569:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078028.883577:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769078028.883579:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617462400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078028.883580:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078028.883589:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5018 00000100:00080000:3.0:1769078028.883596:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078028.883605:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078028.883611:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078028.883618:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617462656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078028.883624:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078028.883627:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078028.883628:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078028.883632:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5018 00000100:00080000:3.0:1769078028.883634:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078028.883636:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078028.883641:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078028.883648:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617462784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078028.883655:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078028.883657:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078028.883658:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078028.883661:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5018 00000100:00080000:3.0:1769078028.883691:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078028.883696:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078028.883701:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078028.883709:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078028.883721:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617462912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078028.883765:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078028/real 1769078028] req@ffff9b0352a18a80 x1855007617462400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078028.883787:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617462400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078028.883800:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078028.883805:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078028.883810:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617462400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078028.883838:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078028.883863:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078028.883875:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078028/real 1769078028] req@ffff9b0352a1b100 x1855007617462528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078028.883889:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617462528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078028.883898:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078028.883902:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078028.883906:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078028.883913:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078028.883934:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078028.883945:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078028/real 1769078028] req@ffff9b0352a1ad80 x1855007617462656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078028.883965:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617462656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078028.883974:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078028.883977:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078028.883981:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078028.883988:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078028.884021:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078028.884033:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078028/real 1769078028] req@ffff9b0352a1b480 x1855007617462784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078028.884046:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617462784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078028.884054:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078028.884057:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078028.884061:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078028.884067:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078028.884086:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078028.884107:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078028/real 1769078028] req@ffff9b0352a1a680 x1855007617462912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078028.884120:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617462912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078028.884135:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078028.884139:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078028.884143:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617462912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078029.395411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078029.780082:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078029.780090:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078029.780221:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078030.419307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078030.803326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078030.803340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078030.803419:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078031.443411:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078031.827336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078031.827336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078031.827448:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078032.467342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078032.851337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078032.851399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078032.851404:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078033.491389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078033.875325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078033.875433:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078033.875495:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078034.003539:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078034.003556:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078034.003562:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078034.003575:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5023 00000100:00080000:0.0:1769078034.003588:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078034.003596:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078034.003656:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617463040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078034.003693:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078034.003698:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078034.003700:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078034.003705:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5023 00000100:00080000:0.0:1769078034.003709:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078034.003714:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078034.003739:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078034.003757:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617463168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078034.003765:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078034.003769:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078034.003771:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078034.003775:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5023 00000100:00080000:0.0:1769078034.003779:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078034.003782:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078034.003788:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078034.003799:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617463296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078034.003808:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078034.003812:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078034.003813:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078034.003817:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5023 00000100:00080000:0.0:1769078034.003820:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078034.003825:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078034.003830:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078034.003839:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617463424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078034.003847:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078034.003850:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078034.003852:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078034.003856:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5023 00000100:00080000:0.0:1769078034.003858:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078034.003862:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078034.003866:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078034.003874:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617463552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078034.003933:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617463040:192.168.204.160@tcp:250:kworker.0 00000800:00000100:2.0:1769078034.004016:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078034.004063:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9b0365a3aa00 x1855007617463040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078034.004094:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617463040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078034.004106:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078034.004111:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078034.004115:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617463040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078034.004124:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078034.004149:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078034.004164:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9b0365a3b480 x1855007617463168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078034.004184:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617463168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078034.004193:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078034.004196:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078034.004200:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078034.004206:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078034.004225:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078034.004259:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9b0365a38e00 x1855007617463296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078034.004271:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617463296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078034.004278:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078034.004281:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078034.004284:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078034.004290:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078034.004311:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078034.004322:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9b0365a39c00 x1855007617463424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078034.004332:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617463424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078034.004362:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078034.004365:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078034.004369:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078034.004375:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078034.004394:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078034.004405:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078034/real 1769078034] req@ffff9b0365a38a80 x1855007617463552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078034.004412:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617463552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078034.004419:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078034.004423:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078034.004426:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078034.515418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078034.899399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078034.899406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078034.899462:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078035.539387:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078035.923345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078035.923364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078035.923413:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078036.563371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078036.947345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078036.947345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078036.947394:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078037.587465:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078037.971311:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078037.971317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078037.971380:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078038.611359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078038.995347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078038.995347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078038.995410:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078039.123335:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078039.123343:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078039.123346:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078039.123354:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5028 00000100:00080000:3.0:1769078039.123362:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078039.123368:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078039.123396:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617463680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078039.123415:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078039.123418:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078039.123419:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078039.123422:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5028 00000100:00080000:3.0:1769078039.123424:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078039.123426:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078039.123433:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078039.123440:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617463808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078039.123447:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078039.123451:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769078039.123452:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617463680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078039.123453:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078039.123459:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5028 00000100:00080000:3.0:1769078039.123460:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078039.123463:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078039.123465:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078039.123471:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617463936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078039.123476:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078039.123478:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078039.123479:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078039.123482:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5028 00000100:00080000:3.0:1769078039.123484:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078039.123486:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078039.123489:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078039.123496:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617464064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078039.123501:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078039.123503:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078039.123504:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078039.123506:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5028 00000100:00080000:3.0:1769078039.123515:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078039.123517:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078039.123520:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078039.123527:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078039.123529:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617464192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078039.123561:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078039/real 1769078039] req@ffff9b0352a18380 x1855007617463680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078039.123578:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617463680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078039.123585:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078039.123588:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078039.123591:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617463680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078039.123606:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078039.123620:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078039.123627:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078039/real 1769078039] req@ffff9b0352a19c00 x1855007617463808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078039.123634:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617463808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078039.123639:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078039.123641:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078039.123643:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078039.123647:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078039.123655:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078039.123660:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078039/real 1769078039] req@ffff9b0352a1b800 x1855007617463936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078039.123666:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617463936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078039.123675:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078039.123676:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078039.123678:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617463936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078039.123682:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078039.123700:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078039.123706:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078039/real 1769078039] req@ffff9b0352a19500 x1855007617464064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078039.123715:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617464064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078039.123722:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078039.123725:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078039.123728:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078039.123733:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078039.123751:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078039.123761:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078039/real 1769078039] req@ffff9b0352a18000 x1855007617464192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078039.123772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617464192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078039.123777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078039.123778:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078039.123781:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078039.635349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078040.019305:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078040.019313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078040.019390:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078040.659346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078041.043321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078041.043321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078041.043341:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078041.683358:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078042.067393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078042.067393:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078042.067474:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078042.707353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078043.091364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078043.091419:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078043.091471:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078043.731348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078044.115345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078044.115368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078044.115408:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078044.243382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078044.243393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078044.243398:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078044.243407:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5033 00000100:00080000:3.0:1769078044.243418:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078044.243424:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078044.243461:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617464320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078044.243490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078044.243494:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078044.243496:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078044.243501:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5033 00000100:00080000:3.0:1769078044.243505:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078044.243510:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078044.243520:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078044.243528:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617464320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078044.243537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617464448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078044.243547:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078044.243549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078044.243551:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078044.243554:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5033 00000100:00080000:3.0:1769078044.243557:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078044.243560:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078044.243563:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078044.243571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617464576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078044.243577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078044.243579:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078044.243580:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078044.243584:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5033 00000100:00080000:3.0:1769078044.243603:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078044.243608:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078044.243611:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078044.243618:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617464704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078044.243624:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078044.243626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078044.243627:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078044.243631:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5033 00000100:00080000:3.0:1769078044.243636:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078044.243638:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078044.243641:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078044.243650:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617464832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078044.243675:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078044.243704:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078044.243724:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078044.243742:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078044.755302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078045.139341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078045.139341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078045.139398:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078045.779292:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078046.163329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078046.163329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078046.163349:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078046.803324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078047.187409:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078047.187410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078047.187442:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1769078047.315598:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:1.0:1769078047.315658:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1769078047.315682:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769078047.315740:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769078047.315750:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769078047.315758:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:1.0:1769078047.315766:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769078047.316069:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078044/real 1769078047] req@ffff9b0352a19500 x1855007617464448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078047.316103:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617464448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078047.316129:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078047.316214:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078047.316219:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078047.316229:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078044/real 1769078047] req@ffff9b0352a1b800 x1855007617464576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078047.316283:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617464576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078047.316296:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078047.316300:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078047.316303:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078047.316310:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078044/real 1769078047] req@ffff9b0352a19c00 x1855007617464704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078047.316320:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617464704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078047.316326:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078047.316329:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078047.316331:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078047.316339:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078044/real 1769078047] req@ffff9b0352a18380 x1855007617464832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078047.316350:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617464832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078047.316357:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078047.316360:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078047.316363:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617464832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078047.316369:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078044/real 1769078047] req@ffff9b0352a18000 x1855007617464320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078047.316381:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617464320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078047.316387:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078047.316390:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078047.316393:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617464320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078047.827313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078048.211338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078048.211338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078048.211399:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078048.851529:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078049.235343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078049.235343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078049.235426:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078049.363357:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078049.363371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078049.363377:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078049.363389:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5038 00000100:00080000:3.0:1769078049.363401:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078049.363410:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078049.363447:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617464960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078049.363476:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078049.363480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078049.363481:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078049.363484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5038 00000100:00080000:3.0:1769078049.363486:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078049.363490:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078049.363499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078049.363508:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617465088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078049.363517:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617464960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078049.363518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078049.363527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078049.363530:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078049.363535:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5038 00000100:00080000:3.0:1769078049.363538:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078049.363542:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078049.363548:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078049.363561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617465216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078049.363570:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078049.363573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078049.363575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078049.363580:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5038 00000100:00080000:3.0:1769078049.363595:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078049.363600:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078049.363605:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078049.363617:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078049.363618:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617465344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078049.363630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078049.363634:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078049.363636:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078049.363641:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5038 00000100:00080000:3.0:1769078049.363645:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078049.363649:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078049.363654:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078049.363680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617465472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078049.363686:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078049/real 1769078049] req@ffff9b0352a1a680 x1855007617464960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078049.363720:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617464960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078049.363732:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078049.363737:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078049.363741:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617464960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078049.363788:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078049.363814:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078049.363827:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078049/real 1769078049] req@ffff9b0352a1b480 x1855007617465088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078049.363840:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617465088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078049.363850:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078049.363854:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078049.363858:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078049.363864:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078049.363884:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078049.363895:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078049/real 1769078049] req@ffff9b0352a1ad80 x1855007617465216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078049.363907:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617465216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078049.363914:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078049.363918:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078049.363921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078049.363927:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078049.363950:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078049.363968:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078049/real 1769078049] req@ffff9b0352a1b100 x1855007617465344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078049.363981:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617465344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078049.363988:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078049.363992:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078049.363996:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078049.364001:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078049.364021:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078049.364031:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078049/real 1769078049] req@ffff9b0352a18a80 x1855007617465472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078049.364041:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617465472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078049.364049:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078049.364052:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078049.364055:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078049.875275:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078050.259350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078050.259357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078050.259410:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078050.899295:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078051.283345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078051.283348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078051.283376:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078051.923300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078052.307412:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078052.307412:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078052.307468:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078052.947310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078053.331379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078053.331389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078053.331432:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078053.971311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078054.355386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078054.355387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078054.355442:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078054.483321:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078054.483330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078054.483334:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078054.483343:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5043 00000100:00080000:3.0:1769078054.483354:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078054.483362:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078054.483396:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617465600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078054.483420:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078054.483423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078054.483425:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078054.483428:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5043 00000100:00080000:3.0:1769078054.483430:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078054.483433:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078054.483441:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078054.483449:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617465728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078054.483457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078054.483460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078054.483461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078054.483466:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5043 00000100:00100000:2.0:1769078054.483473:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617465600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078054.483474:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078054.483489:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078054.483497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078054.483513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617465856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078054.483525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078054.483528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078054.483531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078054.483537:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5043 00000100:00080000:3.0:1769078054.483557:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078054.483563:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078054.483571:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078054.483576:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078054.483592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617465984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078054.483603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078054.483606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078054.483608:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078054.483611:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5043 00000100:00080000:3.0:1769078054.483617:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078054.483621:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078054.483627:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078054.483630:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078054/real 1769078054] req@ffff9b0352a1aa00 x1855007617465600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078054.483642:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617466112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078054.483652:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617465600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078054.483663:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078054.483668:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078054.483681:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617465600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078054.483707:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078054.483738:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078054.483749:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078054/real 1769078054] req@ffff9b0352a18e00 x1855007617465728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078054.483757:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617465728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078054.483764:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078054.483766:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078054.483768:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078054.483773:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078054.483786:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078054.483792:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078054/real 1769078054] req@ffff9b0352a1a300 x1855007617465856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078054.483799:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617465856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078054.483805:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078054.483806:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078054.483808:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078054.483812:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078054.483823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078054.483830:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078054/real 1769078054] req@ffff9b0352a19f80 x1855007617465984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078054.483837:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617465984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078054.483842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078054.483844:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078054.483846:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617465984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078054.483850:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617466112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078054.483859:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078054.483865:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078054/real 1769078054] req@ffff9b0352a18700 x1855007617466112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078054.483872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617466112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078054.483877:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078054.483879:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078054.483881:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617466112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078054.995363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078055.379334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078055.379346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078055.379418:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078056.019375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078056.403369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078056.403369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078056.403426:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078057.043369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078057.427329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078057.427329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078057.427385:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078058.067364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078058.451362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078058.451362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078058.451412:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078059.091363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078059.475388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078059.475388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078059.475457:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078059.603373:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078059.603382:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078059.603386:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078059.603395:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5048 00000100:00080000:3.0:1769078059.603406:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078059.603412:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078059.603444:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617466240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078059.603469:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078059.603472:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078059.603473:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078059.603476:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5048 00000100:00080000:3.0:1769078059.603479:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078059.603482:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078059.603496:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078059.603509:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617466368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078059.603521:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617466240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078059.603527:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078059.603536:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078059.603539:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078059.603544:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5048 00000100:00080000:3.0:1769078059.603549:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078059.603555:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078059.603559:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078059.603570:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617466496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078059.603578:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078059.603580:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078059.603581:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078059.603586:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5048 00000100:00080000:3.0:1769078059.603606:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078059.603611:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078059.603617:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078059.603628:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078059.603634:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617466624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078059.603648:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078059.603651:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078059.603652:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078059.603657:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5048 00000100:00080000:3.0:1769078059.603661:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078059.603665:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078059.603669:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078059.603679:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617466752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078059.603703:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078059/real 1769078059] req@ffff9b0352a18700 x1855007617466240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078059.603727:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617466240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078059.603739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078059.603745:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078059.603750:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617466240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078059.603782:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617466368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078059.603810:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078059.603824:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078059/real 1769078059] req@ffff9b0352a19f80 x1855007617466368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078059.603838:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617466368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078059.603849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078059.603853:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078059.603856:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617466368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078059.603864:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617466496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078059.603894:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078059.603905:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078059/real 1769078059] req@ffff9b0352a1a300 x1855007617466496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078059.603945:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617466496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078059.603953:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078059.603957:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078059.603961:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617466496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078059.603968:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617466624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078059.604014:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078059.604027:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078059/real 1769078059] req@ffff9b0352a18e00 x1855007617466624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078059.604040:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617466624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078059.604049:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078059.604053:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078059.604056:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617466624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078059.604063:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617466752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078059.604084:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078059.604096:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078059/real 1769078059] req@ffff9b0352a1aa00 x1855007617466752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078059.604108:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617466752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078059.604124:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078059.604129:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078059.604132:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617466752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078060.115346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078060.499362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078060.499380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078060.499421:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078061.139334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078061.523377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078061.523377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078061.523439:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078062.163390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078062.547370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078062.547370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078062.547432:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078063.187376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078063.571328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078063.571329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078063.571395:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078064.211378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078064.595406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078064.595407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078064.595461:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078064.723430:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078064.723443:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078064.723448:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078064.723458:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5053 00000100:00080000:0.0:1769078064.723469:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078064.723476:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078064.723510:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617466880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078064.723536:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078064.723539:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078064.723541:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078064.723544:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5053 00000100:00080000:0.0:1769078064.723547:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078064.723550:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078064.723558:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078064.723568:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617467008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078064.723577:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078064.723580:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078064.723581:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078064.723583:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5053 00000100:00080000:0.0:1769078064.723584:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078064.723597:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078064.723599:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617466880:192.168.204.160@tcp:250:kworker.0 00000008:00100000:0.0:1769078064.723601:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078064.723619:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617467136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078064.723627:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078064.723629:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078064.723631:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078064.723634:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5053 00000100:00080000:0.0:1769078064.723637:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078064.723639:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078064.723643:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078064.723652:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617467264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078064.723668:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078064.723673:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078064.723676:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078064.723681:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5053 00000100:00080000:0.0:1769078064.723686:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078064.723692:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769078064.723696:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769078064.723705:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078064.723730:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617467392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078064.723755:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078064/real 1769078064] req@ffff9b0365a38a80 x1855007617466880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078064.723776:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617466880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078064.723786:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078064.723790:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078064.723794:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617466880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078064.723858:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078064.723880:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078064.723890:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078064/real 1769078064] req@ffff9b0365a39c00 x1855007617467008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078064.723904:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617467008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078064.723911:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078064.723914:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078064.723916:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078064.723921:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078064.723932:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078064.723938:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078064/real 1769078064] req@ffff9b0365a38e00 x1855007617467136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078064.723945:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617467136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078064.723950:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078064.723952:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078064.723954:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078064.723958:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078064.723969:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078064.723975:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078064/real 1769078064] req@ffff9b0365a3b480 x1855007617467264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078064.723982:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617467264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078064.723992:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078064.723994:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078064.723996:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078064.724000:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078064.724009:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078064.724015:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078064/real 1769078064] req@ffff9b0365a3aa00 x1855007617467392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078064.724022:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617467392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078064.724027:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078064.724029:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078064.724031:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078065.235301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078065.619353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078065.619374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078065.619426:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078066.259349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078066.643400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078066.643400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078066.643410:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078067.283308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078067.667389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078067.667397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078067.667458:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078068.307335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078068.691395:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078068.691396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078068.691477:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078069.331301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078069.715356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078069.715397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078069.715439:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078069.843326:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078069.843340:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078069.843346:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078069.843358:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5059 00000100:00080000:0.0:1769078069.843372:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078069.843382:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078069.843413:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617467520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078069.843442:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078069.843446:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078069.843448:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078069.843452:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5059 00000100:00080000:0.0:1769078069.843455:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078069.843458:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078069.843467:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078069.843480:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617467648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078069.843486:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617467520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078069.843494:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078069.843497:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078069.843500:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078069.843504:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5059 00000100:00080000:0.0:1769078069.843508:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078069.843511:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078069.843517:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078069.843531:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617467776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078069.843539:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078069.843543:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078069.843544:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078069.843549:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5059 00000800:00000100:3.0:1769078069.843554:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769078069.843556:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078069.843563:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078069.843570:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078069.843582:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617467904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078069.843590:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078069.843592:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078069.843594:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078069.843598:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5059 00000100:00080000:0.0:1769078069.843600:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078069.843602:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078069.843605:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078069.843611:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617468032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078069.843654:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078069/real 1769078069] req@ffff9b0365a39180 x1855007617467520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078069.843671:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617467520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078069.843679:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078069.843682:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078069.843685:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617467520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078069.843704:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078069.843845:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078069.843859:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078069/real 1769078069] req@ffff9b0365a39500 x1855007617467648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078069.843872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617467648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078069.843932:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078069.843934:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078069.843937:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078069.843942:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078069.843958:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078069.843966:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078069/real 1769078069] req@ffff9b0365a3bb80 x1855007617467776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078069.843974:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617467776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078069.843984:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078069.843986:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078069.843988:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078069.843992:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078069.844001:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078069.844007:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078069/real 1769078069] req@ffff9b0365a3ad80 x1855007617467904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078069.844014:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617467904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078069.844019:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078069.844021:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078069.844023:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617467904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078069.844026:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078069.844035:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078069.844040:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078069/real 1769078069] req@ffff9b0365a3b100 x1855007617468032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078069.844051:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617468032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078069.844059:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078069.844062:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078069.844066:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078070.355343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078070.739380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078070.739387:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078070.739436:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078071.379353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078071.763365:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078071.763392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078071.763441:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078072.403365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078072.787317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078072.787330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078072.787423:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078073.427395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078073.811372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078073.811375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078073.811466:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078074.451375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078074.835329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078074.835340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078074.835423:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078074.963510:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078074.963526:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078074.963531:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078074.963546:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5064 00000100:00080000:3.0:1769078074.963557:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078074.963566:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078074.963612:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617468160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078074.963691:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617468160:192.168.204.160@tcp:250:kworker.0 00000800:00000100:2.0:1769078074.963773:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078074.963810:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078074.963813:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078074.963815:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078074.963819:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5064 00000100:00080000:3.0:1769078074.963823:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078074.963827:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078074.963836:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078074.963852:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617468288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078074.963924:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078074/real 1769078074] req@ffff9b0352a18a80 x1855007617468160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078074.963944:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617468160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078074.963956:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078074.963961:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078074.963966:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617468160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078074.963996:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078074.964024:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078074.964038:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078074/real 1769078074] req@ffff9b0352a1b100 x1855007617468288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078074.964051:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617468288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078074.964062:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078074.964066:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078074.964070:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468288:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769078074.964135:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078074.964140:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078074.964143:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078074.964148:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5064 00000100:00080000:3.0:1769078074.964155:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078074.964162:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078074.964168:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078074.964183:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617468416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078074.964203:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078074.964207:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078074.964210:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078074.964216:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5064 00000100:00080000:3.0:1769078074.964220:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078074.964224:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468416:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769078074.964229:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078074.964264:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078074.964272:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078074.964282:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617468544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078074.964288:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078074/real 1769078074] req@ffff9b0352a1ad80 x1855007617468416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078074.964306:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078074.964313:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769078074.964317:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617468416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078074.964317:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078074.964324:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5064 00000100:00080000:3.0:1769078074.964329:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078074.964335:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078074.964337:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769078074.964344:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000008:00100000:3.0:1769078074.964347:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078074.964350:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078074.964366:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617468672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078074.964372:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078074.964405:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078074.964418:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078074/real 1769078074] req@ffff9b0352a1b480 x1855007617468544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078074.964432:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617468544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078074.964442:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078074.964446:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078074.964449:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078074.964463:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078074.964485:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078074.964497:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078074/real 1769078074] req@ffff9b0352a1a680 x1855007617468672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078074.964511:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617468672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078074.964527:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078074.964531:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078074.964535:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078075.475367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078075.859370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078075.859379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078075.859417:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078076.499381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078076.883369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078076.883369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078076.883393:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078077.523366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078077.907360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078077.907360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078077.907428:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078078.547391:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078078.931369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078078.931369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078078.931447:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078079.571359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078079.955354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078079.955355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078079.955383:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078080.083282:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078080.083291:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078080.083295:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078080.083302:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5069 00000100:00080000:3.0:1769078080.083312:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078080.083318:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078080.083353:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617468800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078080.083385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078080.083389:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078080.083391:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078080.083395:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5069 00000100:00080000:3.0:1769078080.083400:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078080.083405:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078080.083416:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078080.083428:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617468800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078080.083432:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617468928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078080.083451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078080.083456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078080.083459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078080.083464:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5069 00000100:00080000:3.0:1769078080.083468:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078080.083473:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078080.083480:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078080.083491:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617469056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078080.083501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078080.083504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078080.083506:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078080.083510:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5069 00000800:00000100:2.0:1769078080.083514:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078080.083521:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078080.083528:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078080.083535:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078080.083547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617469184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078080.083555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078080.083559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769078080.083588:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078080/real 1769078080] req@ffff9b0352a18000 x1855007617468800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078080.083588:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078080.083603:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5069 00000100:00080000:3.0:1769078080.083615:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078080.083622:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078080.083628:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617468800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078080.083630:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769078080.083648:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078080.083650:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617469312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078080.083656:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078080.083672:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617468800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078080.083701:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078080.083721:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078080.083729:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078080/real 1769078080] req@ffff9b0352a18380 x1855007617468928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078080.083738:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617468928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078080.083744:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078080.083747:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078080.083749:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617468928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078080.083753:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078080.083763:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078080.083768:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078080/real 1769078080] req@ffff9b0352a19c00 x1855007617469056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078080.083775:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617469056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078080.083780:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078080.083781:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078080.083783:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078080.083786:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078080.083796:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078080.083801:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078080/real 1769078080] req@ffff9b0352a1b800 x1855007617469184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078080.083808:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617469184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078080.083812:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078080.083819:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078080.083823:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078080.083827:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078080.083837:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078080.083842:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078080/real 1769078080] req@ffff9b0352a19500 x1855007617469312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078080.083849:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617469312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078080.083854:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078080.083856:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078080.083858:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078080.595328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078080.979307:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078080.979347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078080.979389:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078081.619409:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078082.003313:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078082.003313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078082.003343:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078082.643352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078083.027388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078083.027390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078083.027497:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078083.667357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078084.051354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078084.051391:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078084.051393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078084.691328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078085.075331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078085.075331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078085.075351:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078085.203375:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078085.203385:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078085.203389:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078085.203399:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5074 00000100:00080000:0.0:1769078085.203410:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078085.203418:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078085.203461:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617469440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078085.203496:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078085.203502:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078085.203503:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078085.203507:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5074 00000100:00080000:0.0:1769078085.203510:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078085.203513:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078085.203523:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078085.203534:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617469568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078085.203539:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617469440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078085.203544:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078085.203547:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078085.203548:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078085.203552:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5074 00000100:00080000:0.0:1769078085.203554:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078085.203557:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078085.203560:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078085.203567:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617469696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078085.203573:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078085.203575:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078085.203577:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078085.203579:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5074 00000100:00080000:0.0:1769078085.203582:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078085.203584:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078085.203587:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078085.203593:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617469824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078085.203598:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078085.203600:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078085.203601:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078085.203603:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5074 00000100:00080000:0.0:1769078085.203606:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078085.203609:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769078085.203612:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078085.203616:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078085.203624:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617469952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078085.203654:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078085/real 1769078085] req@ffff9b0365a3b100 x1855007617469440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078085.203669:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617469440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078085.203677:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078085.203680:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078085.203684:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617469440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078085.203705:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078085.203718:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078085.203725:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078085/real 1769078085] req@ffff9b0365a3ad80 x1855007617469568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078085.203733:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617469568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078085.203738:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078085.203740:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078085.203742:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078085.203745:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078085.203753:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078085.203758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078085/real 1769078085] req@ffff9b0365a3bb80 x1855007617469696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078085.203764:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617469696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078085.203769:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078085.203779:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078085.203782:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078085.203787:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078085.203805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078085.203816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078085/real 1769078085] req@ffff9b0365a39500 x1855007617469824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078085.203827:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617469824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078085.203835:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078085.203837:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078085.203839:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078085.203843:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078085.203856:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078085.203862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078085/real 1769078085] req@ffff9b0365a39180 x1855007617469952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078085.203869:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617469952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078085.203874:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078085.203876:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078085.203878:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617469952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078085.715363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078086.099374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078086.099375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078086.099432:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078086.739309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078087.123369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078087.123403:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078087.123403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078087.763382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078088.147351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078088.147357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078088.147437:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078088.787383:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078089.171350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078089.171361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078089.171380:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078089.811354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078090.195350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078090.195350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078090.195405:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078090.323361:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078090.323370:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078090.323374:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078090.323382:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5079 00000100:00080000:3.0:1769078090.323410:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078090.323417:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078090.323449:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617470080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078090.323478:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078090.323481:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078090.323482:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078090.323486:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5079 00000100:00080000:3.0:1769078090.323489:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078090.323492:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078090.323501:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078090.323511:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617470208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078090.323516:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617470080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078090.323522:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078090.323525:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078090.323527:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078090.323531:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5079 00000100:00080000:3.0:1769078090.323535:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078090.323538:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078090.323541:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078090.323550:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617470336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078090.323557:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078090.323559:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078090.323561:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078090.323563:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5079 00000100:00080000:3.0:1769078090.323566:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078090.323569:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078090.323572:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078090.323581:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617470464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078090.323589:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078090.323593:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078090.323596:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078090.323598:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078090.323601:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5079 00000100:00080000:3.0:1769078090.323604:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078090.323607:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078090.323610:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078090.323617:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617470592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078090.323635:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078090/real 1769078090] req@ffff9b0352a19500 x1855007617470080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078090.323649:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617470080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078090.323656:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078090.323659:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078090.323662:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617470080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078090.323682:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078090.323707:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078090.323720:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078090/real 1769078090] req@ffff9b0352a1b800 x1855007617470208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078090.323732:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617470208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078090.323738:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078090.323740:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078090.323742:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078090.323746:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078090.323760:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078090.323767:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078090/real 1769078090] req@ffff9b0352a19c00 x1855007617470336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078090.323785:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617470336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078090.323795:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078090.323797:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078090.323799:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078090.323803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078090.323814:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078090.323821:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078090/real 1769078090] req@ffff9b0352a18380 x1855007617470464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078090.323828:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617470464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078090.323833:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078090.323834:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078090.323836:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078090.323840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078090.323850:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078090.323855:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078090/real 1769078090] req@ffff9b0352a18000 x1855007617470592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078090.323862:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617470592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078090.323867:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078090.323869:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078090.323871:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078090.835367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078091.219684:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078091.219923:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078091.219956:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078091.859355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078092.243384:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078092.243433:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078092.243578:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078092.883339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078093.267454:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078093.267734:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078093.268150:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078093.907343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078094.291767:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078094.292015:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078094.292105:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078094.931352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078095.315463:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078095.315517:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078095.315747:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078095.443352:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078095.443361:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078095.443366:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078095.443374:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5084 00000100:00080000:0.0:1769078095.443382:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078095.443388:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078095.443426:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617470720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078095.443452:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078095.443457:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078095.443460:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078095.443463:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5084 00000100:00080000:0.0:1769078095.443467:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078095.443470:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078095.443479:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078095.443492:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617470848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078095.443492:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617470720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078095.443507:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078095.443509:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078095.443511:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078095.443514:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5084 00000100:00080000:0.0:1769078095.443516:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078095.443519:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078095.443522:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078095.443531:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617470976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078095.443543:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078095.443546:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078095.443547:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078095.443549:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5084 00000800:00000100:2.0:1769078095.443554:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769078095.443558:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078095.443563:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078095.443567:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078095.443577:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617471104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078095.443583:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078095.443585:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078095.443587:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078095.443589:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5084 00000100:00080000:0.0:1769078095.443592:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078095.443595:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078095.443598:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078095.443600:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078095/real 1769078095] req@ffff9b0365a3aa00 x1855007617470720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078095.443608:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617471232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078095.443635:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617470720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078095.443648:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078095.443653:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078095.443657:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617470720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078095.443681:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078095.443706:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078095.443720:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078095/real 1769078095] req@ffff9b0365a3b480 x1855007617470848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078095.443734:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617470848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078095.443744:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078095.443747:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078095.443749:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078095.443754:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078095.443770:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078095.443778:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078095/real 1769078095] req@ffff9b0365a38e00 x1855007617470976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078095.443786:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617470976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078095.443791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078095.443794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078095.443796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617470976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078095.443799:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078095.443809:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078095.443814:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078095/real 1769078095] req@ffff9b0365a39c00 x1855007617471104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078095.443821:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617471104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078095.443827:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078095.443828:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078095.443830:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078095.443834:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078095.443843:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078095.443849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078095/real 1769078095] req@ffff9b0365a38a80 x1855007617471232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078095.443861:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617471232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078095.443867:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078095.443870:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078095.443873:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078095.955341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078096.339407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078096.339568:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078096.339669:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078096.979309:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078097.363363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078097.363363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078097.363400:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078098.003348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078098.387355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078098.387355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078098.387407:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078099.027307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078099.411346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078099.411363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078099.411431:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078100.051386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078100.435388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078100.435422:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078100.435472:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078100.563362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078100.563374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078100.563379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078100.563388:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5089 00000100:00080000:3.0:1769078100.563397:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078100.563403:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078100.563434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617471360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078100.563457:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078100.563459:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078100.563461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078100.563464:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5089 00000100:00080000:3.0:1769078100.563468:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078100.563473:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078100.563484:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078100.563488:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617471360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078100.563503:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617471488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078100.563517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078100.563522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078100.563524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078100.563528:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5089 00000100:00080000:3.0:1769078100.563532:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078100.563535:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078100.563541:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078100.563545:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078100.563556:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617471616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078100.563564:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078100.563567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078100.563568:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078100.563571:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5089 00000100:00080000:3.0:1769078100.563575:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078100.563577:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078100.563581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078100.563583:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078100/real 1769078100] req@ffff9b0352a1a680 x1855007617471360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078100.563592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617471744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078100.563602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078100.563606:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617471360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078100.563606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078100.563608:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078100.563616:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5089 00000100:00080000:3.0:1769078100.563621:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078100.563625:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769078100.563626:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769078100.563631:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769078100.563634:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078100.563640:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617471360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078100.563643:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617471872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078100.563665:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078100.563681:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078100.563690:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078100/real 1769078100] req@ffff9b0352a1b480 x1855007617471488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078100.563699:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617471488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078100.563705:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078100.563707:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078100.563710:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078100.563713:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078100.563723:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078100.563728:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078100/real 1769078100] req@ffff9b0352a1ad80 x1855007617471616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078100.563736:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617471616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078100.563741:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078100.563742:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078100.563744:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078100.563748:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078100.563757:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078100.563762:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078100/real 1769078100] req@ffff9b0352a1b100 x1855007617471744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078100.563769:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617471744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078100.563775:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078100.563777:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078100.563779:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078100.563782:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078100.563791:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078100.563796:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078100/real 1769078100] req@ffff9b0352a18a80 x1855007617471872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078100.563802:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617471872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078100.563807:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078100.563809:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078100.563811:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617471872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078101.075367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078101.459354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078101.459357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078101.459412:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078102.099368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078102.483348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078102.483348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078102.483413:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078103.123339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078103.507377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078103.507378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078103.507440:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078104.147321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078104.531345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078104.531345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078104.531394:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078105.171335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078105.555356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078105.555504:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078105.555521:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769078105.683335:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078105.683347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078105.683352:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078105.683363:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5094 00000100:00080000:3.0:1769078105.683374:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078105.683393:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078105.683430:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617472000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078105.683459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078105.683463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078105.683468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078105.683473:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5094 00000100:00080000:3.0:1769078105.683478:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078105.683483:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078105.683493:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078105.683509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617472128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078105.683522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078105.683526:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617472000:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078105.683526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078105.683530:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078105.683535:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5094 00000100:00080000:3.0:1769078105.683540:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078105.683545:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078105.683551:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078105.683562:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617472256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078105.683572:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078105.683576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078105.683578:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078105.683583:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5094 00000100:00080000:3.0:1769078105.683587:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078105.683589:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078105.683631:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078105/real 1769078105] req@ffff9b0352a1aa00 x1855007617472000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078105.683649:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617472000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078105.683656:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:0.0:1769078105.683659:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1769078105.683661:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1769078105.683664:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078105.683667:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617472000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078105.683672:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617472384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078105.683681:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078105.683684:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078105.683686:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078105.683688:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472128:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769078105.683689:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5094 00000100:00080000:3.0:1769078105.683693:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078105.683697:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078105.683701:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769078105.683710:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078105.683711:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617472512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078105.683723:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078105/real 1769078105] req@ffff9b0352a18e00 x1855007617472128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078105.683746:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617472128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078105.683755:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078105.683758:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078105.683762:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078105.683768:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078105.683788:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078105.683799:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078105/real 1769078105] req@ffff9b0352a1a300 x1855007617472256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078105.683812:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617472256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078105.683827:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078105.683831:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078105.683834:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078105.683840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078105.683862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078105.683874:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078105/real 1769078105] req@ffff9b0352a19f80 x1855007617472384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078105.683887:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617472384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078105.683897:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078105.683900:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078105.683904:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078105.683920:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078105.683940:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078105.683951:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078105/real 1769078105] req@ffff9b0352a18700 x1855007617472512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078105.683964:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617472512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078105.683971:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078105.683973:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078105.683976:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078106.195303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078106.579343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078106.579343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078106.579438:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078107.219353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078107.603339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078107.603354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078107.603407:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078108.243305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078108.627338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078108.627338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078108.627395:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078109.267349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078109.651343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078109.651355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078109.651412:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078110.291394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769078110.675533:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078110.675626:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078110.675686:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:1.0:1769078110.803390:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078110.803400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078110.803404:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078110.803412:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5099 00000100:00080000:1.0:1769078110.803421:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078110.803427:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078110.803458:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617472640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078110.803484:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078110.803487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078110.803488:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078110.803491:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5099 00000100:00080000:1.0:1769078110.803494:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078110.803496:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078110.803503:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078110.803512:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617472768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078110.803519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078110.803523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078110.803524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078110.803527:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5099 00000100:00080000:1.0:1769078110.803530:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078110.803535:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078110.803537:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617472640:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769078110.803539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078110.803550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617472896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078110.803557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078110.803559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078110.803561:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078110.803564:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5099 00000100:00080000:1.0:1769078110.803566:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078110.803569:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078110.803572:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078110.803582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617473024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078110.803588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078110.803590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078110.803592:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078110.803595:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5099 00000100:00080000:1.0:1769078110.803597:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078110.803599:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078110.803603:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078110.803610:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617473152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078110.803632:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078110.803664:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078110.803691:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078110.803709:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078111.315282:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078111.699353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078111.699358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078111.699436:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078112.339287:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078112.723341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078112.723392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078112.723414:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078113.363304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078113.747300:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078113.747303:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078113.747364:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769078113.875652:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769078113.875696:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769078113.875714:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078113.875759:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078113.875771:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078113.875784:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769078113.875795:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078110/real 1769078113] req@ffff9b0343ded880 x1855007617472896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1769078113.875805:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769078113.875832:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617472896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078113.875844:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078113.875849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078113.875854:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078113.875861:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078110/real 1769078113] req@ffff9b0343def800 x1855007617473024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078113.875873:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617473024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078113.875880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078113.875883:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078113.875886:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078113.875893:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078110/real 1769078113] req@ffff9b0343deea00 x1855007617473152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078113.875903:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617473152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078113.875910:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078113.875929:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078113.875932:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078113.875939:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078110/real 1769078113] req@ffff9b0343def100 x1855007617472640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078113.875953:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617472640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078113.875962:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078113.875965:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078113.875969:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617472640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078113.875980:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078110/real 1769078113] req@ffff9b0343dedf80 x1855007617472768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078113.875992:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617472768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078113.876016:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078113.876020:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078113.876024:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617472768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078114.387318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078114.771338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078114.771338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078114.771388:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078115.412666:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078115.795334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078115.795334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078115.795387:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078115.923366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078115.923378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078115.923382:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078115.923392:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5105 00000100:00080000:0.0:1769078115.923401:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078115.923408:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078115.923440:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617473280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078115.923475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078115.923479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078115.923481:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078115.923485:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5105 00000100:00080000:0.0:1769078115.923488:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078115.923492:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078115.923501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078115.923512:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617473408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078115.923521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078115.923525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078115.923526:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078115.923530:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5105 00000100:00080000:0.0:1769078115.923533:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078115.923537:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078115.923541:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078115.923549:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617473536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078115.923558:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078115.923561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078115.923563:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078115.923566:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5105 00000100:00080000:0.0:1769078115.923569:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078115.923572:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078115.923577:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078115.923586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617473664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078115.923593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078115.923596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078115.923598:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078115.923601:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5105 00000100:00080000:0.0:1769078115.923603:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078115.923607:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078115.923611:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078115.923619:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617473792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078115.923695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617473280:192.168.204.160@tcp:250:kworker.0 00000800:00000100:2.0:1769078115.923763:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078115.923799:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078115/real 1769078115] req@ffff9b0365a38a80 x1855007617473280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078115.923816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617473280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078115.923826:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078115.923831:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078115.923835:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617473280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078115.923842:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078115.923863:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078115.923875:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078115/real 1769078115] req@ffff9b0365a39c00 x1855007617473408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078115.923886:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617473408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078115.923894:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078115.923897:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078115.923901:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078115.923907:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078115.923923:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078115.923933:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078115/real 1769078115] req@ffff9b0365a38e00 x1855007617473536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078115.923943:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617473536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078115.923950:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078115.923953:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078115.923956:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078115.923977:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078115.923997:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078115.924008:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078115/real 1769078115] req@ffff9b0365a3b480 x1855007617473664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078115.924019:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617473664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078115.924027:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078115.924030:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078115.924033:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078115.924038:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078115.924055:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078115.924065:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078115/real 1769078115] req@ffff9b0365a3aa00 x1855007617473792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078115.924074:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617473792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078115.924082:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078115.924084:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078115.924087:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617473792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078116.435305:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078116.819365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078116.819374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078116.819434:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078117.459331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078117.843371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078117.843372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078117.843425:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078118.483337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078118.867330:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078118.867342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078118.867371:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078119.507343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078119.891360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078119.891365:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078119.891439:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078120.531389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078120.915350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078120.915351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078120.915431:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078121.043362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.043374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078121.043379:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.043389:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5110 00000100:00080000:0.0:1769078121.043400:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.043407:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078121.043448:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617473920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078121.043475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.043478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078121.043479:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.043482:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5110 00000100:00080000:0.0:1769078121.043484:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.043487:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078121.043495:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078121.043507:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617474048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078121.043517:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617473920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078121.043522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.043532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078121.043534:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.043537:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5110 00000100:00080000:0.0:1769078121.043539:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.043542:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078121.043545:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078121.043561:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617474176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078121.043567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.043569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078121.043570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.043573:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5110 00000100:00080000:0.0:1769078121.043582:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.043585:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769078121.043592:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769078121.043595:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078121.043608:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617474304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078121.043614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078121.043617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078121.043619:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078121.043622:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5110 00000100:00080000:0.0:1769078121.043626:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078121.043630:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078121.043634:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078121.043634:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9b0365a39180 x1855007617473920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078121.043653:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617474432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078121.043661:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617473920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.043671:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.043674:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078121.043677:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617473920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078121.043699:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078121.043714:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078121.043720:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9b0365a39500 x1855007617474048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078121.043727:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617474048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.043732:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.043734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078121.043736:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078121.043740:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078121.043748:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078121.043753:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9b0365a3bb80 x1855007617474176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078121.043763:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617474176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.043767:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.043769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078121.043770:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078121.043773:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078121.043782:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078121.043787:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9b0365a3ad80 x1855007617474304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078121.043792:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617474304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.043796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.043798:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078121.043800:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078121.043803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078121.043811:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078121.043816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078121/real 1769078121] req@ffff9b0365a3b100 x1855007617474432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078121.043821:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617474432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078121.043829:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078121.043831:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078121.043833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078121.555370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078121.939400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078121.939400:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078121.939485:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078122.579380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078122.963338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078122.963338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078122.963395:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078123.603355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078123.987301:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078123.987307:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078123.987358:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078124.627332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078125.011348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078125.011379:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078125.011442:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078125.651310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078126.035347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078126.035347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078126.035395:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078126.163357:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078126.163368:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078126.163373:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078126.163383:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5115 00000100:00080000:2.0:1769078126.163393:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078126.163400:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078126.163437:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617474560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078126.163469:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078126.163473:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078126.163475:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078126.163480:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5115 00000100:00080000:2.0:1769078126.163484:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078126.163487:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078126.163497:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078126.163510:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617474688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078126.163522:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078126.163528:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617474560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769078126.163528:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078126.163532:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078126.163543:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5115 00000100:00080000:2.0:1769078126.163546:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078126.163550:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078126.163555:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078126.163567:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617474816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078126.163585:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078126.163588:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078126.163591:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078126.163595:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5115 00000100:00080000:2.0:1769078126.163599:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078126.163605:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769078126.163605:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078126.163614:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078126.163626:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617474944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078126.163635:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078126.163639:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078126.163641:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078126.163645:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5115 00000100:00080000:2.0:1769078126.163650:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078126.163656:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078126/real 1769078126] req@ffff9b03482c4700 x1855007617474560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078126.163657:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078126.163664:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078126.163680:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617475072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078126.163689:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617474560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078126.163704:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078126.163709:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078126.163715:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617474560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078126.163741:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078126.163766:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078126.163779:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078126/real 1769078126] req@ffff9b03482c4e00 x1855007617474688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078126.163812:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617474688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078126.163822:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078126.163826:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078126.163830:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078126.163842:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078126.163863:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078126.163875:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078126/real 1769078126] req@ffff9b03482c7b80 x1855007617474816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078126.163889:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617474816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078126.163897:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078126.163901:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078126.163905:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078126.163911:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078126.163931:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078126.163942:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078126/real 1769078126] req@ffff9b03482c5f80 x1855007617474944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078126.163954:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617474944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078126.163963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078126.163966:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078126.163970:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617474944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078126.163976:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078126.163993:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078126.164004:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078126/real 1769078126] req@ffff9b03482c5c00 x1855007617475072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078126.164016:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617475072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078126.164025:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078126.164028:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078126.164031:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078126.675342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078127.059373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078127.059398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078127.059423:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078127.699359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078128.083391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078128.083392:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078128.083457:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078128.723366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078129.107342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078129.107342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078129.107423:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078129.747335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078130.131381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078130.131381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078130.131449:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078130.771366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078131.155380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078131.155399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078131.155456:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078131.283380:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078131.283405:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078131.283411:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078131.283426:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5120 00000100:00080000:2.0:1769078131.283438:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078131.283449:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078131.283501:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617475200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078131.283538:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078131.283543:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078131.283546:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078131.283551:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5120 00000100:00080000:2.0:1769078131.283556:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078131.283561:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078131.283574:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078131.283589:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617475200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078131.283606:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617475328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078131.283621:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078131.283626:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078131.283628:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078131.283634:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5120 00000100:00080000:2.0:1769078131.283639:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078131.283644:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078131.283649:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078131.283665:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617475456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078131.283673:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769078131.283683:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078131.283688:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078131.283691:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078131.283696:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5120 00000100:00080000:2.0:1769078131.283718:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078131.283724:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078131.283731:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078131.283746:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617475584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078131.283757:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078131/real 1769078131] req@ffff9b03482c5880 x1855007617475200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078131.283759:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078131.283767:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078131.283781:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078131.283789:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5120 00000100:00100000:0.0:1769078131.283791:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617475200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078131.283796:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078131.283806:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:0.0:1769078131.283808:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1769078131.283814:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1769078131.283815:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078131.283822:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617475200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078131.283836:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617475712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078131.283849:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078131.283873:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078131.283884:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078131/real 1769078131] req@ffff9b03482c5c00 x1855007617475328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078131.283893:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617475328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078131.283899:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078131.283902:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078131.283904:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078131.283908:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078131.283919:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078131.283925:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078131/real 1769078131] req@ffff9b03482c5f80 x1855007617475456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078131.283933:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617475456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078131.283939:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078131.283940:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078131.283942:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078131.283947:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078131.283958:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078131.283964:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078131/real 1769078131] req@ffff9b03482c7b80 x1855007617475584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078131.283972:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617475584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078131.283977:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078131.283979:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078131.283981:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078131.283991:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078131.284001:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078131.284007:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078131/real 1769078131] req@ffff9b03482c4e00 x1855007617475712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078131.284014:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617475712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078131.284020:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078131.284021:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078131.284023:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078131.795368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078132.179394:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078132.179394:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078132.179442:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078132.819371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078133.203303:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078133.203327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078133.203384:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078133.843370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078134.227355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078134.227364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078134.227415:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078134.867348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078135.251309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078135.251356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078135.251397:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078135.891362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078136.275339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078136.275339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078136.275431:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078136.403376:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078136.403385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078136.403389:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078136.403397:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5125 00000100:00080000:1.0:1769078136.403406:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078136.403411:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078136.403444:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617475840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078136.403470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078136.403473:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078136.403475:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078136.403478:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5125 00000100:00080000:1.0:1769078136.403480:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078136.403482:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078136.403489:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078136.403500:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617475968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078136.403508:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078136.403511:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078136.403513:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078136.403515:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5125 00000100:00080000:1.0:1769078136.403518:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078136.403520:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078136.403523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078136.403532:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617475840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078136.403537:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617476096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078136.403561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078136.403565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078136.403566:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078136.403569:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5125 00000100:00080000:1.0:1769078136.403571:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078136.403573:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078136.403577:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078136.403588:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617476224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078136.403595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078136.403598:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078136.403600:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078136.403602:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5125 00000100:00080000:1.0:1769078136.403632:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078136.403637:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078136.403642:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1769078136.403649:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078136.403656:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617476352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078136.403709:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078136/real 1769078136] req@ffff9b0343deca80 x1855007617475840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078136.403727:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617475840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078136.403745:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078136.403749:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078136.403752:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617475840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078136.403759:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078136.403779:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078136.403788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078136/real 1769078136] req@ffff9b0343deed80 x1855007617475968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078136.403797:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617475968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078136.403803:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078136.403806:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078136.403808:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617475968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078136.403832:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078136.403844:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078136.403850:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078136/real 1769078136] req@ffff9b0343ded500 x1855007617476096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078136.403858:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617476096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078136.403869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078136.403871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078136.403874:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078136.403877:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078136.403888:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078136.403894:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078136/real 1769078136] req@ffff9b0343def480 x1855007617476224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078136.403901:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617476224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078136.403906:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078136.403908:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078136.403910:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078136.403914:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078136.403924:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078136.403930:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078136/real 1769078136] req@ffff9b0343dee680 x1855007617476352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078136.403937:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617476352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078136.403942:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078136.403944:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078136.403946:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078136.915338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078137.299351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078137.299351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078137.299377:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078137.939341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078138.323323:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078138.323325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078138.323411:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078138.963342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078139.347351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078139.347351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078139.347416:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078139.987336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078140.371317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078140.371327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078140.371374:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078141.011342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078141.395343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078141.395343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078141.395421:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078141.523341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078141.523349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078141.523353:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078141.523361:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5130 00000100:00080000:1.0:1769078141.523369:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078141.523377:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078141.523407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617476480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078141.523429:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078141.523432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078141.523434:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078141.523437:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5130 00000100:00080000:1.0:1769078141.523440:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078141.523443:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078141.523451:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078141.523461:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617476608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078141.523469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078141.523472:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078141.523473:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078141.523477:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5130 00000100:00080000:1.0:1769078141.523480:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078141.523483:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078141.523488:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078141.523497:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617476480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078141.523501:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617476736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078141.523513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078141.523516:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078141.523517:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078141.523521:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5130 00000100:00080000:1.0:1769078141.523524:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078141.523541:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078141.523546:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078141.523558:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617476864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078141.523565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078141.523568:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078141.523570:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078141.523573:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5130 00000100:00080000:1.0:1769078141.523576:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078141.523578:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078141.523582:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078141.523587:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078141.523595:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617476992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078141.523614:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078141/real 1769078141] req@ffff9b0343defb80 x1855007617476480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078141.523630:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617476480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078141.523638:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078141.523642:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078141.523646:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617476480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078141.523653:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078141.523672:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078141.523681:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078141/real 1769078141] req@ffff9b0343dece00 x1855007617476608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078141.523689:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617476608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078141.523695:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078141.523697:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078141.523699:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078141.523717:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078141.523729:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078141.523735:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078141/real 1769078141] req@ffff9b0343dee680 x1855007617476736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078141.523742:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617476736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078141.523747:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078141.523749:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078141.523751:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078141.523754:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078141.523764:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078141.523770:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078141/real 1769078141] req@ffff9b0343def480 x1855007617476864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078141.523777:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617476864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078141.523782:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078141.523784:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078141.523786:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078141.523789:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078141.523798:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078141.523804:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078141/real 1769078141] req@ffff9b0343ded500 x1855007617476992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078141.523816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617476992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078141.523821:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078141.523822:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078141.523824:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617476992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078142.035344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078142.419340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078142.419340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078142.419380:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078143.059375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078143.443374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078143.443388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078143.443478:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078144.083354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078144.467334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078144.467335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078144.467404:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078145.107338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078145.491359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078145.491360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078145.491500:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078146.131381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078146.515353:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078146.515364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078146.515426:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078146.643397:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078146.643423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078146.643429:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078146.643464:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5135 00000100:00080000:0.0:1769078146.643478:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078146.643492:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078146.643544:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617477120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078146.643577:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078146.643581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078146.643583:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078146.643589:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5135 00000100:00080000:0.0:1769078146.643594:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078146.643600:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078146.643611:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078146.643624:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617477248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078146.643634:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617477120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078146.643635:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078146.643639:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078146.643641:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078146.643645:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5135 00000100:00080000:0.0:1769078146.643649:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078146.643652:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078146.643657:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078146.643666:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617477376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078146.643674:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078146.643677:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078146.643679:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078146.643683:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5135 00000100:00080000:0.0:1769078146.643685:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078146.643689:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078146.643695:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078146.643704:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617477504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078146.643712:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078146.643715:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078146.643717:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078146.643721:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5135 00000800:00000100:2.0:1769078146.643723:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769078146.643725:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078146.643728:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078146.643740:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078146.643750:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617477632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078146.643781:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078146/real 1769078146] req@ffff9b0365a3b100 x1855007617477120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078146.643801:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617477120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078146.643818:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078146.643822:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078146.643825:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617477120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078146.643851:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617477248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078146.643875:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078146.643893:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078146/real 1769078146] req@ffff9b0365a3ad80 x1855007617477248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078146.643906:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617477248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078146.643913:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078146.643916:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078146.643920:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617477248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078146.643925:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617477376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078146.643944:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078146.643954:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078146/real 1769078146] req@ffff9b0365a3bb80 x1855007617477376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078146.643966:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617477376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078146.643974:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078146.643978:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078146.643981:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617477376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078146.643987:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617477504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078146.644005:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078146.644014:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078146/real 1769078146] req@ffff9b0365a39500 x1855007617477504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078146.644026:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617477504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078146.644034:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078146.644037:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078146.644040:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617477504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078146.644045:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617477632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078146.644074:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078146.644086:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078146/real 1769078146] req@ffff9b0365a39180 x1855007617477632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078146.644098:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617477632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078146.644106:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078146.644109:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078146.644112:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617477632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078147.155328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078147.539345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078147.539346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078147.539398:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078148.179378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078148.563312:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078148.563347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078148.563377:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078149.203337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078149.587342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078149.587342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078149.587390:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078150.227341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078150.611300:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078150.611302:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078150.611349:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078151.251349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078151.635342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078151.635342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078151.635424:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078151.763335:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078151.763345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078151.763351:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078151.763380:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5140 00000100:00080000:0.0:1769078151.763390:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078151.763397:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078151.763434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617477760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078151.763465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078151.763469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078151.763471:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078151.763476:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5140 00000100:00080000:0.0:1769078151.763480:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078151.763485:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078151.763495:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078151.763509:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617477888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078151.763517:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617477760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078151.763542:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078151.763546:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078151.763549:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078151.763554:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5140 00000100:00080000:0.0:1769078151.763558:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078151.763563:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078151.763569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078151.763582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617478016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078151.763593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078151.763597:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078151.763602:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078151.763602:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769078151.763610:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5140 00000100:00080000:0.0:1769078151.763615:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078151.763619:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078151.763624:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078151.763636:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617478144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078151.763645:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078151.763651:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769078151.763651:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078151/real 1769078151] req@ffff9b0365a3aa00 x1855007617477760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078151.763655:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078151.763664:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5140 00000100:00080000:0.0:1769078151.763670:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078151.763674:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078151.763680:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078151.763685:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617477760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078151.763694:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617478272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078151.763702:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078151.763710:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078151.763716:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617477760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078151.763741:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617477888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078151.763765:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078151.763778:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078151/real 1769078151] req@ffff9b0365a3b480 x1855007617477888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078151.763790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617477888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078151.763799:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078151.763803:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078151.763806:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617477888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078151.763812:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078151.763832:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078151.763844:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078151/real 1769078151] req@ffff9b0365a38e00 x1855007617478016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078151.763863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617478016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078151.763871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078151.763875:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078151.763878:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078151.763884:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078151.763902:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078151.763910:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078151/real 1769078151] req@ffff9b0365a39c00 x1855007617478144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078151.763918:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617478144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078151.763923:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078151.763925:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078151.763927:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078151.763931:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078151.763940:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078151.763945:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078151/real 1769078151] req@ffff9b0365a38a80 x1855007617478272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078151.763952:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617478272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078151.763961:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078151.763963:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078151.763965:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078152.275334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078152.659317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078152.659317:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078152.659388:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078153.299354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078153.683357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078153.683364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078153.683376:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078154.323356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078154.707341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078154.707397:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078154.707414:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078155.347354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078155.731350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078155.731350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078155.731413:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078156.371356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078156.755391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078156.755392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078156.755464:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078156.883432:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078156.883447:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078156.883453:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078156.883466:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5146 00000100:00080000:3.0:1769078156.883478:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078156.883486:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078156.883548:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617478400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078156.883580:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078156.883584:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078156.883586:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078156.883591:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5146 00000100:00080000:3.0:1769078156.883594:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078156.883599:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078156.883608:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078156.883621:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617478528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078156.883631:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078156.883634:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078156.883635:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078156.883640:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5146 00000100:00080000:3.0:1769078156.883646:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078156.883649:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617478400:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078156.883653:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078156.883667:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078156.883683:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617478656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078156.883694:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078156.883697:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078156.883698:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078156.883701:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5146 00000100:00080000:3.0:1769078156.883703:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078156.883705:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078156.883710:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078156.883717:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617478784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078156.883724:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078156.883727:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078156.883729:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078156.883737:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078156.883743:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5146 00000100:00080000:3.0:1769078156.883750:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078156.883755:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078156.883761:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078156.883786:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078156/real 1769078156] req@ffff9b0352a18700 x1855007617478400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078156.883787:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617478912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078156.883821:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617478400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078156.883835:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078156.883842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078156.883847:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617478400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078156.883876:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078156.883903:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078156.883917:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078156/real 1769078156] req@ffff9b0352a19f80 x1855007617478528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078156.883931:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617478528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078156.883947:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078156.883951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078156.883955:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078156.883962:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078156.883984:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078156.883997:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078156/real 1769078156] req@ffff9b0352a1a300 x1855007617478656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078156.884010:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617478656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078156.884019:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078156.884023:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078156.884026:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078156.884034:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078156.884072:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078156.884085:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078156/real 1769078156] req@ffff9b0352a18e00 x1855007617478784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078156.884099:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617478784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078156.884108:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078156.884111:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078156.884116:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078156.884123:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078156.884143:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078156.884153:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078156/real 1769078156] req@ffff9b0352a1aa00 x1855007617478912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078156.884165:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617478912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078156.884174:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078156.884178:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078156.884181:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617478912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078157.395379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078157.779406:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078157.779418:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078157.779536:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078158.419413:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078158.803483:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078158.803590:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078158.803656:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078159.443378:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078159.827402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078159.827902:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078159.827904:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078160.467388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078160.851781:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078160.851916:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078160.852277:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078161.491373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078161.875450:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078161.875664:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078161.875691:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078162.003331:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.003342:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078162.003348:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.003358:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5151 00000100:00080000:3.0:1769078162.003367:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.003373:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078162.003403:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617479040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078162.003425:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.003428:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078162.003430:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.003432:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5151 00000100:00080000:3.0:1769078162.003435:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.003437:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078162.003444:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078162.003453:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617479168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078162.003461:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.003465:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078162.003466:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.003469:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5151 00000100:00100000:2.0:1769078162.003475:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617479040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078162.003475:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.003482:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078162.003486:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078162.003495:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617479296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078162.003502:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.003505:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078162.003506:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.003509:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5151 00000100:00080000:3.0:1769078162.003511:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.003513:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078162.003516:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078162.003523:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617479424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078162.003529:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078162.003531:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078162.003533:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078162.003535:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5151 00000800:00000100:2.0:1769078162.003540:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078162.003541:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078162.003546:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078162.003549:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078162.003557:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617479552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078162.003597:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff9b0352a18a80 x1855007617479040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.003612:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617479040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.003620:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.003624:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078162.003628:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617479040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078162.003636:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078162.003651:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078162.003659:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff9b0352a1b100 x1855007617479168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.003667:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617479168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.003672:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.003675:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078162.003677:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078162.003695:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078162.003705:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078162.003711:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff9b0352a1ad80 x1855007617479296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.003718:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617479296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.003723:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.003725:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078162.003727:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078162.003731:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078162.003740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078162.003746:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff9b0352a1b480 x1855007617479424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.003754:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617479424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.003758:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.003760:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078162.003762:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078162.003765:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078162.003774:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078162.003779:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078162/real 1769078162] req@ffff9b0352a1a680 x1855007617479552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078162.003786:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617479552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078162.003791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078162.003793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078162.003795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078162.515331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078162.899778:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078162.899841:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078162.900034:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078163.539365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078163.923328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078163.923328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078163.923383:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078164.563308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078164.947355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078164.947368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078164.947384:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078165.587354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078165.971316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078165.971316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078165.971368:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078166.611345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078166.995328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078166.995329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078166.995361:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078167.123386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078167.123406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078167.123411:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078167.123421:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5156 00000100:00080000:1.0:1769078167.123431:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078167.123439:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078167.123480:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617479680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078167.123518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078167.123522:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078167.123524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078167.123528:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5156 00000100:00080000:1.0:1769078167.123531:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078167.123534:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078167.123542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078167.123554:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617479808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078167.123567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078167.123570:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617479680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078167.123575:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078167.123579:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078167.123585:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5156 00000100:00080000:1.0:1769078167.123590:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078167.123596:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078167.123602:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078167.123617:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617479936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078167.123630:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078167.123639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078167.123643:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078167.123646:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078167.123650:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5156 00000100:00080000:1.0:1769078167.123655:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078167.123658:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078167.123663:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078167.123668:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078167/real 1769078167] req@ffff9b0343deed80 x1855007617479680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078167.123676:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617480064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078167.123686:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078167.123689:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078167.123692:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078167.123692:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617479680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078167.123700:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5156 00000100:00080000:1.0:1769078167.123705:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078167.123708:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078167.123710:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078167.123716:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617480192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078167.123726:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078167.123734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078167.123741:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617479680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078167.123763:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078167.123784:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078167.123792:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078167/real 1769078167] req@ffff9b0343deca80 x1855007617479808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078167.123802:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617479808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078167.123808:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078167.123810:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078167.123813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078167.123817:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078167.123827:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078167.123833:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078167/real 1769078167] req@ffff9b0343dedf80 x1855007617479936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078167.123840:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617479936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078167.123850:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078167.123852:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078167.123854:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617479936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078167.123857:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078167.123867:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078167.123873:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078167/real 1769078167] req@ffff9b0343def100 x1855007617480064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078167.123883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617480064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078167.123891:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078167.123895:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078167.123898:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078167.123904:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078167.123924:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078167.123935:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078167/real 1769078167] req@ffff9b0343deea00 x1855007617480192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078167.123947:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617480192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078167.123953:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078167.123955:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078167.123957:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078167.635354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078168.019339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078168.019339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078168.019414:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078168.659346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078169.043301:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078169.043369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078169.043396:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078169.683389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078170.067398:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078170.067398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078170.067465:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078170.707323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078171.091346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078171.091357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078171.091449:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078171.731340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078172.115293:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078172.115298:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078172.115370:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078172.243354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078172.243366:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078172.243370:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078172.243381:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5161 00000100:00080000:1.0:1769078172.243391:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078172.243399:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078172.243437:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617480320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078172.243480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078172.243485:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078172.243487:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078172.243492:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5161 00000100:00080000:1.0:1769078172.243496:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078172.243501:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078172.243512:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078172.243530:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617480448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078172.243530:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617480320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078172.243548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078172.243552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078172.243555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078172.243560:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5161 00000100:00080000:1.0:1769078172.243565:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078172.243569:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078172.243575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078172.243588:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617480576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078172.243599:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078172.243602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078172.243606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078172.243609:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078172.243615:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5161 00000100:00080000:1.0:1769078172.243622:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078172.243626:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078172.243632:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078172.243648:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078172/real 1769078172] req@ffff9b0343def800 x1855007617480320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078172.243649:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617480704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078172.243689:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617480320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078172.243692:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078172.243701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078172.243705:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078172.243714:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5161 00000100:00080000:2.0:1769078172.243714:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078172.243720:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078172.243723:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:1.0:1769078172.243728:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078172.243731:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617480320:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769078172.243735:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078172.243747:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617480832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078172.243756:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078172.243781:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078172.243794:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078172/real 1769078172] req@ffff9b0343ded880 x1855007617480448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078172.243814:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617480448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078172.243824:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078172.243828:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078172.243833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078172.243839:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078172.243858:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078172.243867:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078172/real 1769078172] req@ffff9b0343dee300 x1855007617480576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078172.243875:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617480576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078172.243880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078172.243882:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078172.243885:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078172.243889:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078172.243900:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078172.243906:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078172/real 1769078172] req@ffff9b0343dec380 x1855007617480704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078172.243913:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617480704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078172.243918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078172.243920:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078172.243922:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078172.243933:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078172.243942:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078172.243948:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078172/real 1769078172] req@ffff9b0343ded180 x1855007617480832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078172.243956:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617480832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078172.243963:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078172.243966:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078172.243969:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617480832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078172.755343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078173.139324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078173.139324:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078173.139387:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078173.779338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078174.163304:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078174.163306:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078174.163390:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078174.803349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078175.187404:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078175.187404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078175.187462:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078175.827364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078176.211372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078176.211423:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078176.211436:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078176.851367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078177.235335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078177.235335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078177.235396:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078177.363349:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078177.363362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078177.363367:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078177.363377:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5166 00000100:00080000:1.0:1769078177.363388:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078177.363397:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078177.363455:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617480960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078177.363484:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078177.363488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078177.363491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078177.363495:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5166 00000100:00080000:1.0:1769078177.363500:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078177.363505:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078177.363516:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078177.363528:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617480960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078177.363532:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617481088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078177.363546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078177.363550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078177.363552:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078177.363557:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5166 00000100:00080000:1.0:1769078177.363562:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078177.363566:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078177.363573:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078177.363588:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617481216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078177.363599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078177.363602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078177.363605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078177.363610:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5166 00000100:00080000:1.0:1769078177.363616:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078177.363621:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078177.363627:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078177.363641:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617481344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078177.363651:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078177.363656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078177.363658:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078177.363663:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481088:192.168.204.160@tcp:38:kworker.0 00000100:00080000:1.0:1769078177.363666:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5166 00000100:00080000:1.0:1769078177.363674:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078177.363677:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078177.363682:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078177.363694:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617481472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078177.363702:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078177.363721:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078177.363736:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078177.875290:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078178.259349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078178.259350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078178.259518:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078178.899327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078179.283329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078179.283329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078179.283381:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078179.923298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078180.307337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078180.307367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078180.307422:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769078180.435584:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769078180.435626:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769078180.435642:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078180.435680:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078180.435688:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078180.435695:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078180.435701:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769078180.435728:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078177/real 1769078180] req@ffff9b0343dee300 x1855007617481472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078180.435754:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617481472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078180.435765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078180.435769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078180.435773:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078180.435780:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078177/real 1769078180] req@ffff9b0343dec700 x1855007617481088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078180.435798:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617481088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078180.435807:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078180.435811:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078180.435813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078180.435820:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078177/real 1769078180] req@ffff9b0343ded180 x1855007617481216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078180.435830:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617481216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078180.435835:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078180.435838:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078180.435841:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078180.435846:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078177/real 1769078180] req@ffff9b0343dec380 x1855007617481344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078180.435855:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617481344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078180.435861:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078180.435864:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078180.435867:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078180.435882:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078177/real 1769078180] req@ffff9b0343dec000 x1855007617480960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078180.435896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617480960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078180.435903:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078180.435906:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078180.435908:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617480960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078180.947287:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078181.331334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078181.331335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078181.331400:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078181.971627:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078182.355339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078182.355411:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078182.355434:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078182.483402:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078182.483413:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078182.483417:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078182.483427:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5171 00000100:00080000:3.0:1769078182.483435:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078182.483441:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078182.483478:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617481600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078182.483504:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078182.483507:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078182.483509:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078182.483513:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5171 00000100:00080000:3.0:1769078182.483517:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078182.483522:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078182.483532:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078182.483547:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617481728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078182.483562:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078182.483567:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769078182.483569:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617481600:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078182.483570:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078182.483578:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5171 00000100:00080000:3.0:1769078182.483582:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078182.483586:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078182.483591:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078182.483604:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617481856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078182.483612:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078182.483616:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078182.483618:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078182.483621:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5171 00000100:00080000:3.0:1769078182.483632:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078182.483636:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078182.483641:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078182.483649:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078182.483654:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617481984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078182.483664:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078182.483667:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078182.483670:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078182.483673:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5171 00000100:00080000:3.0:1769078182.483678:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078182.483681:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078182.483686:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078182.483692:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078182/real 1769078182] req@ffff9b0352a18000 x1855007617481600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078182.483700:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617482112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078182.483724:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617481600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078182.483738:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078182.483743:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078182.483748:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617481600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078182.483774:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078182.483797:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078182.483810:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078182/real 1769078182] req@ffff9b0352a18380 x1855007617481728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078182.483822:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617481728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078182.483828:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078182.483830:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078182.483833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078182.483849:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078182.483870:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078182.483885:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078182/real 1769078182] req@ffff9b0352a19c00 x1855007617481856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078182.483901:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617481856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078182.483909:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078182.483913:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078182.483916:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078182.483921:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078182.483939:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078182.483949:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078182/real 1769078182] req@ffff9b0352a1b800 x1855007617481984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078182.483960:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617481984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078182.483966:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078182.483968:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078182.483971:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617481984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078182.483975:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617482112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078182.483995:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078182.484005:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078182/real 1769078182] req@ffff9b0352a19500 x1855007617482112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078182.484020:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617482112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078182.484029:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078182.484033:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078182.484036:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617482112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078182.995294:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078183.379326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078183.379363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078183.379410:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078184.020004:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078184.403392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078184.403400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078184.403464:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078185.043307:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078185.427309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078185.427319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078185.427352:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078186.067300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078186.451351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078186.451359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078186.451415:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078187.091333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078187.475333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078187.475333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078187.475380:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078187.603339:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078187.603348:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078187.603351:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078187.603359:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5176 00000100:00080000:1.0:1769078187.603367:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078187.603372:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078187.603400:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617482240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078187.603427:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078187.603430:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078187.603431:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078187.603434:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5176 00000100:00080000:1.0:1769078187.603436:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078187.603439:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078187.603445:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078187.603453:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617482368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078187.603460:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078187.603464:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078187.603466:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078187.603469:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5176 00000100:00080000:1.0:1769078187.603474:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078187.603475:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617482240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078187.603482:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078187.603488:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078187.603502:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617482496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078187.603509:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078187.603512:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078187.603513:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078187.603516:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5176 00000100:00080000:1.0:1769078187.603519:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078187.603521:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078187.603524:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078187.603533:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617482624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078187.603538:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078187.603541:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078187.603542:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078187.603546:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078187.603547:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5176 00000100:00080000:1.0:1769078187.603551:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078187.603554:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078187.603556:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078187.603563:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617482752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078187.603589:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078187/real 1769078187] req@ffff9b0343ded880 x1855007617482240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078187.603605:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617482240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078187.603615:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078187.603619:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078187.603623:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617482240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078187.603630:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617482368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078187.603650:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078187.603658:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078187/real 1769078187] req@ffff9b0343def800 x1855007617482368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078187.603666:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617482368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078187.603671:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078187.603673:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078187.603675:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617482368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078187.603691:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617482496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078187.603701:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078187.603707:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078187/real 1769078187] req@ffff9b0343deea00 x1855007617482496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078187.603713:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617482496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078187.603718:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078187.603719:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078187.603721:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617482496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078187.603724:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617482624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078187.603733:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078187.603738:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078187/real 1769078187] req@ffff9b0343def100 x1855007617482624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078187.603744:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617482624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078187.603748:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078187.603750:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078187.603766:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617482624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078187.603770:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617482752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078187.603782:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078187.603788:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078187/real 1769078187] req@ffff9b0343dedf80 x1855007617482752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078187.603795:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617482752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078187.603799:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078187.603801:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078187.603803:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617482752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078188.115335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078188.499326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078188.499326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078188.499377:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078189.139331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078189.523322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078189.523328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078189.523365:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078190.163337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078190.547344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078190.547375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078190.547411:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078191.187325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078191.571294:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078191.571294:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078191.571364:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078192.211349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078192.595328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078192.595328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078192.595395:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078192.723314:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078192.723323:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078192.723327:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078192.723335:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5181 00000100:00080000:1.0:1769078192.723345:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078192.723354:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078192.723389:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617482880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078192.723414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078192.723417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078192.723418:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078192.723421:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5181 00000100:00080000:1.0:1769078192.723424:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078192.723427:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078192.723434:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078192.723443:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617483008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078192.723451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078192.723455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769078192.723456:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617482880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078192.723457:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078192.723461:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5181 00000100:00080000:1.0:1769078192.723466:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078192.723469:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078192.723471:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078192.723478:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617483136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078192.723485:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078192.723488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078192.723489:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078192.723493:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5181 00000100:00080000:1.0:1769078192.723496:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078192.723499:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078192.723504:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078192.723515:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617483264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078192.723525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1769078192.723529:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078192.723532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078192.723535:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078192.723539:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5181 00000100:00080000:1.0:1769078192.723549:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078192.723552:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078192.723556:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078192.723567:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617483392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078192.723568:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078192/real 1769078192] req@ffff9b0343deca80 x1855007617482880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078192.723594:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617482880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078192.723605:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078192.723610:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078192.723613:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617482880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078192.723633:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078192.723656:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078192.723680:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078192/real 1769078192] req@ffff9b0343deed80 x1855007617483008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078192.723693:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617483008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078192.723701:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078192.723705:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078192.723708:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078192.723714:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078192.723732:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078192.723740:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078192/real 1769078192] req@ffff9b0343ded500 x1855007617483136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078192.723751:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617483136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078192.723756:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078192.723758:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078192.723760:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078192.723763:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078192.723773:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078192.723779:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078192/real 1769078192] req@ffff9b0343def480 x1855007617483264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078192.723790:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617483264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078192.723797:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078192.723800:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078192.723803:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078192.723808:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078192.723825:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078192.723835:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078192/real 1769078192] req@ffff9b0343dee680 x1855007617483392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078192.723844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617483392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078192.723853:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078192.723855:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078192.723857:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078193.235328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078193.619329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078193.619334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078193.619509:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078194.259341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078194.643345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078194.643345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078194.643407:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078195.283352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078195.667347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078195.667347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078195.667405:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078196.307333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078196.691356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078196.691356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078196.691547:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078197.331323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078197.715341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078197.715359:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078197.715399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:1.0:1769078197.843330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078197.843340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078197.843346:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078197.843356:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5187 00000100:00080000:1.0:1769078197.843365:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078197.843370:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078197.843398:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617483520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078197.843426:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078197.843430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078197.843431:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078197.843436:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5187 00000100:00080000:1.0:1769078197.843439:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078197.843443:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078197.843453:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078197.843466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617483648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078197.843471:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617483520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078197.843477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078197.843481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078197.843482:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078197.843486:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5187 00000100:00080000:1.0:1769078197.843489:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078197.843492:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078197.843496:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078197.843506:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617483776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078197.843513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078197.843516:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078197.843518:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078197.843521:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5187 00000100:00080000:1.0:1769078197.843525:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078197.843529:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078197.843532:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078197.843539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078197.843552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617483904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078197.843562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1769078197.843563:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078197/real 1769078197] req@ffff9b0343dece00 x1855007617483520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078197.843569:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078197.843573:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078197.843598:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617483520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078197.843599:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5187 00000100:00080000:1.0:1769078197.843605:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078197.843612:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078197.843617:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769078197.843618:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078197.843625:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078197.843627:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617484032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078197.843634:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617483520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078197.843660:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078197.843678:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078197.843686:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078197/real 1769078197] req@ffff9b0343defb80 x1855007617483648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078197.843694:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617483648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078197.843700:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078197.843702:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078197.843704:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078197.843708:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078197.843716:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078197.843721:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078197/real 1769078197] req@ffff9b0343dee680 x1855007617483776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078197.843728:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617483776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078197.843732:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078197.843734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078197.843735:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078197.843738:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078197.843747:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078197.843751:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078197/real 1769078197] req@ffff9b0343def480 x1855007617483904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078197.843757:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617483904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078197.843761:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078197.843763:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078197.843765:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617483904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078197.843768:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078197.843777:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078197.843781:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078197/real 1769078197] req@ffff9b0343ded500 x1855007617484032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078197.843787:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617484032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078197.843792:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078197.843794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078197.843795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078198.355343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078198.739328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078198.739328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078198.739395:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078199.379330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078199.763338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078199.763387:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078199.763430:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078200.403335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078200.787332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078200.787395:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078200.787430:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078201.427350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078201.811337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078201.811337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078201.811397:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078202.451341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078202.835333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078202.835333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078202.835355:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078202.963359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078202.963370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078202.963374:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078202.963383:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5192 00000100:00080000:2.0:1769078202.963392:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078202.963398:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078202.963438:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617484160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078202.963469:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078202.963480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078202.963482:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078202.963487:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5192 00000100:00080000:2.0:1769078202.963491:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078202.963521:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078202.963525:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617484160:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769078202.963531:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078202.963546:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617484288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078202.963557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078202.963560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078202.963562:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078202.963566:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5192 00000100:00080000:2.0:1769078202.963571:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078202.963574:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078202.963579:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1769078202.963587:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078202.963591:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617484416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078202.963601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078202.963605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078202.963607:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078202.963610:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5192 00000100:00080000:2.0:1769078202.963614:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078202.963617:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078202.963622:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078202.963627:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078202/real 1769078202] req@ffff9b03482c4700 x1855007617484160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078202.963634:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617484544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078202.963644:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078202.963647:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617484160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078202.963649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078202.963653:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078202.963660:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078202.963660:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5192 00000100:00080000:2.0:1769078202.963666:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078202.963667:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:2.0:1769078202.963673:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078202.963674:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617484160:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769078202.963682:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078202.963693:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617484672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078202.963700:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078202.963727:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078202.963745:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078202/real 1769078202] req@ffff9b03482c6a00 x1855007617484288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078202.963758:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617484288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078202.963767:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078202.963770:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078202.963774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078202.963779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078202.963795:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078202.963804:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078202/real 1769078202] req@ffff9b03482c4000 x1855007617484416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078202.963820:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617484416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078202.963827:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078202.963830:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078202.963833:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078202.963838:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078202.963851:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078202.963859:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078202/real 1769078202] req@ffff9b03482c7100 x1855007617484544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078202.963866:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617484544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078202.963871:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078202.963873:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078202.963874:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078202.963884:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078202.963894:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078202.963899:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078202/real 1769078202] req@ffff9b03482c6680 x1855007617484672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078202.963905:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617484672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078202.963914:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078202.963915:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078202.963917:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078203.475311:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078203.859351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078203.859413:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078203.859554:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078204.499328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078204.883324:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078204.883372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078204.883384:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078205.523331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078205.907326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078205.907326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078205.907372:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078206.547345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078206.931333:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078206.931333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078206.931389:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078207.571339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078207.955348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078207.955348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078207.955371:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078208.083350:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078208.083359:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078208.083363:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078208.083371:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5197 00000100:00080000:0.0:1769078208.083379:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078208.083384:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078208.083434:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617484800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078208.083458:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078208.083461:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078208.083462:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078208.083465:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5197 00000100:00080000:0.0:1769078208.083468:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078208.083472:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078208.083479:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078208.083492:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617484928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078208.083492:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617484800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078208.083506:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078208.083509:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078208.083510:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078208.083513:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5197 00000100:00080000:0.0:1769078208.083516:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078208.083518:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078208.083521:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078208.083529:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617485056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078208.083535:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078208.083538:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078208.083539:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078208.083541:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5197 00000100:00080000:0.0:1769078208.083545:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078208.083547:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078208.083550:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078208.083557:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617485184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078208.083561:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769078208.083568:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078208.083572:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078208.083573:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078208.083577:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5197 00000100:00080000:0.0:1769078208.083581:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078208.083583:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078208.083587:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078208.083596:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617485312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078208.083601:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078208/real 1769078208] req@ffff9b0365a38a80 x1855007617484800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078208.083617:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617484800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078208.083625:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078208.083630:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078208.083633:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617484800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078208.083658:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078208.083673:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078208.083682:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078208/real 1769078208] req@ffff9b0365a39c00 x1855007617484928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078208.083690:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617484928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078208.083695:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078208.083697:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078208.083699:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617484928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078208.083703:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078208.083713:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078208.083719:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078208/real 1769078208] req@ffff9b0365a38e00 x1855007617485056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078208.083726:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617485056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078208.083731:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078208.083732:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078208.083734:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078208.083738:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078208.083747:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078208.083753:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078208/real 1769078208] req@ffff9b0365a3b480 x1855007617485184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078208.083760:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617485184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078208.083765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078208.083766:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078208.083768:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078208.083772:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078208.083780:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078208.083785:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078208/real 1769078208] req@ffff9b0365a3aa00 x1855007617485312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078208.083791:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617485312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078208.083796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078208.083798:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078208.083799:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078208.595336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078208.979320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078208.979339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078208.979399:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078209.619352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078210.003358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078210.003405:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078210.003418:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078210.643335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078211.027297:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078211.027297:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078211.027348:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078211.667327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078212.051351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078212.051401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078212.051424:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078212.691359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078213.075290:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078213.075316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078213.075349:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078213.203383:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078213.203392:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078213.203397:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078213.203407:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5202 00000100:00080000:1.0:1769078213.203414:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078213.203420:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078213.203454:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617485440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078213.203472:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078213.203475:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078213.203476:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078213.203479:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5202 00000100:00080000:1.0:1769078213.203483:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078213.203486:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078213.203493:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078213.203503:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617485568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078213.203511:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617485440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078213.203513:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078213.203520:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078213.203522:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078213.203526:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5202 00000100:00080000:1.0:1769078213.203529:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078213.203532:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078213.203536:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078213.203543:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617485696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078213.203550:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078213.203552:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078213.203553:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078213.203556:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5202 00000100:00080000:1.0:1769078213.203558:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078213.203560:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078213.203563:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078213.203568:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617485824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078213.203573:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078213.203577:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078213.203579:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1769078213.203579:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078213.203586:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5202 00000100:00080000:1.0:1769078213.203589:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078213.203593:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078213.203597:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078213.203608:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617485952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078213.203620:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078213/real 1769078213] req@ffff9b0343deed80 x1855007617485440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078213.203645:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617485440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078213.203656:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078213.203661:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078213.203666:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617485440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078213.203685:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078213.203706:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078213.203716:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078213/real 1769078213] req@ffff9b0343deca80 x1855007617485568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078213.203728:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617485568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078213.203736:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078213.203739:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078213.203742:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078213.203748:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078213.203762:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078213.203770:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078213/real 1769078213] req@ffff9b0343dedf80 x1855007617485696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078213.203788:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617485696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078213.203796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078213.203799:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078213.203803:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078213.203808:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078213.203826:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078213.203836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078213/real 1769078213] req@ffff9b0343def100 x1855007617485824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078213.203848:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617485824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078213.203856:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078213.203860:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078213.203863:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078213.203869:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078213.203887:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078213.203898:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078213/real 1769078213] req@ffff9b0343deea00 x1855007617485952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078213.203911:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617485952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078213.203924:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078213.203928:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078213.203933:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617485952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078213.715403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078214.099389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078214.099389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078214.099454:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078214.739319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078215.123392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078215.123393:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078215.123408:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078215.763347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078216.147369:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078216.147369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078216.147443:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078216.787335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078217.171345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078217.171345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078217.171443:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078217.811351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078218.195337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078218.195337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078218.195376:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078218.323350:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078218.323361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078218.323366:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078218.323376:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5207 00000100:00080000:3.0:1769078218.323384:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078218.323391:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078218.323421:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617486080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078218.323441:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078218.323444:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078218.323445:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078218.323448:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5207 00000100:00080000:3.0:1769078218.323450:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078218.323452:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078218.323458:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078218.323466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617486208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078218.323473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078218.323476:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078218.323477:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078218.323480:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5207 00000100:00080000:3.0:1769078218.323481:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078218.323485:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617486080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078218.323486:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078218.323492:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078218.323499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617486336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078218.323505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078218.323507:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078218.323508:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078218.323510:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5207 00000100:00080000:3.0:1769078218.323513:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078218.323515:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078218.323518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078218.323524:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617486464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078218.323530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078218.323532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078218.323534:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078218.323537:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5207 00000100:00080000:3.0:1769078218.323542:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078218.323545:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078218.323549:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078218.323555:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078218.323568:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617486592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078218.323578:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078218/real 1769078218] req@ffff9b0352a19500 x1855007617486080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078218.323593:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617486080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078218.323603:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078218.323607:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078218.323610:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617486080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078218.323615:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078218.323631:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078218.323643:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078218/real 1769078218] req@ffff9b0352a1b800 x1855007617486208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078218.323655:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617486208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078218.323663:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078218.323667:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078218.323670:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078218.323690:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078218.323705:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078218.323713:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078218/real 1769078218] req@ffff9b0352a19c00 x1855007617486336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078218.323723:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617486336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078218.323731:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078218.323734:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078218.323737:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078218.323743:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078218.323762:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078218.323770:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078218/real 1769078218] req@ffff9b0352a18380 x1855007617486464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078218.323777:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617486464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078218.323782:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078218.323784:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078218.323786:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078218.323789:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078218.323799:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078218.323804:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078218/real 1769078218] req@ffff9b0352a18000 x1855007617486592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078218.323810:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617486592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078218.323823:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078218.323825:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078218.323827:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078218.835388:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078219.219334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078219.219368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078219.219387:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078219.859339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078220.243352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078220.243364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078220.243402:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078220.883381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078221.267337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078221.267372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078221.267497:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078221.907367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078222.291366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078222.291366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078222.291426:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078222.931322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078223.315350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078223.315350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078223.315390:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078223.443340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078223.443349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078223.443353:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078223.443359:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5212 00000100:00080000:1.0:1769078223.443365:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078223.443370:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078223.443406:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617486720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078223.443424:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078223.443427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078223.443428:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078223.443430:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5212 00000100:00080000:1.0:1769078223.443432:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078223.443435:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078223.443442:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078223.443449:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617486848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078223.443455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078223.443458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078223.443459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078223.443462:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5212 00000100:00080000:1.0:1769078223.443464:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078223.443468:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078223.443471:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078223.443476:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617486976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078223.443481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078223.443483:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078223.443484:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078223.443487:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5212 00000100:00080000:1.0:1769078223.443489:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078223.443492:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078223.443494:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078223.443503:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617486720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078223.443504:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617487104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078223.443530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078223.443533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078223.443534:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078223.443536:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5212 00000100:00080000:1.0:1769078223.443538:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078223.443539:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078223.443542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078223.443550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617487232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078223.443616:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078223.443669:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078223/real 1769078223] req@ffff9b0343def800 x1855007617486720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078223.443688:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617486720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078223.443700:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078223.443704:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078223.443709:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617486720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078223.443718:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078223.443738:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078223.443750:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078223/real 1769078223] req@ffff9b0343ded880 x1855007617486848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078223.443762:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617486848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078223.443771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078223.443774:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078223.443777:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078223.443783:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078223.443801:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078223.443810:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078223/real 1769078223] req@ffff9b0343dec000 x1855007617486976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078223.443821:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617486976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078223.443829:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078223.443832:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078223.443835:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617486976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078223.443864:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078223.443882:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078223.443892:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078223/real 1769078223] req@ffff9b0343dec380 x1855007617487104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078223.443903:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617487104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078223.443911:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078223.443914:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078223.443917:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078223.443923:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078223.443939:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078223.443948:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078223/real 1769078223] req@ffff9b0343ded180 x1855007617487232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078223.443958:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617487232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078223.443965:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078223.443968:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078223.443971:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078223.955328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078224.339568:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078224.339674:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078224.339775:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078224.979346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078225.363561:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078225.366637:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078225.366653:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078226.003335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769078226.387901:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078226.387909:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078226.387954:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078227.027366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078227.411343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078227.411343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078227.411416:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078228.051308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078228.435428:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078228.435444:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078228.435557:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078228.563381:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078228.563394:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078228.563399:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078228.563413:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5217 00000100:00080000:2.0:1769078228.563425:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078228.563434:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078228.563487:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617487360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078228.563525:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078228.563529:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078228.563531:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078228.563536:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5217 00000100:00080000:2.0:1769078228.563540:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078228.563544:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078228.563554:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078228.563577:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617487488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078228.563594:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617487360:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769078228.563594:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078228.563605:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078228.563609:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078228.563615:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5217 00000100:00080000:2.0:1769078228.563619:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078228.563624:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078228.563629:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078228.563653:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617487616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078228.563670:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078228.563673:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078228.563675:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078228.563679:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5217 00000800:00000100:0.0:1769078228.563686:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769078228.563688:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078228.563695:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078228.563702:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078228.563714:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617487744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078228.563730:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078228.563733:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078228/real 1769078228] req@ffff9b03482c6d80 x1855007617487360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078228.563736:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078228.563740:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078228.563752:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5217 00000100:00080000:2.0:1769078228.563760:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078228.563766:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078228.563772:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078228.563774:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617487360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078228.563787:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617487872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078228.563788:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078228.563806:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078228.563818:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617487360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078228.563849:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078228.563873:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078228.563885:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078228/real 1769078228] req@ffff9b03482c6300 x1855007617487488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078228.563903:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617487488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078228.563911:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078228.563915:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078228.563918:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078228.563926:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078228.563952:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078228.563973:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078228/real 1769078228] req@ffff9b03482c5500 x1855007617487616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078228.563985:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617487616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078228.563992:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078228.563995:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078228.563998:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078228.564005:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078228.564024:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078228.564034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078228/real 1769078228] req@ffff9b03482c4a80 x1855007617487744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078228.564044:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617487744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078228.564056:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078228.564060:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078228.564092:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078228.564098:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078228.564119:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078228.564129:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078228/real 1769078228] req@ffff9b03482c5180 x1855007617487872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078228.564141:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617487872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078228.564149:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078228.564152:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078228.564155:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617487872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078229.075354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078229.459441:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078229.460076:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078229.462107:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078230.099359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078230.483319:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078230.484099:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078230.484132:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078231.123356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078231.507349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078231.507371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078231.507406:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078232.147340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078232.531332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078232.531332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078232.531397:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078233.171285:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078233.555330:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078233.555363:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078233.555402:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078233.683305:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078233.683315:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078233.683319:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078233.683329:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5222 00000100:00080000:2.0:1769078233.683337:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078233.683344:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078233.683375:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617488000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078233.683401:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078233.683405:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078233.683406:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078233.683410:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5222 00000100:00080000:2.0:1769078233.683413:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078233.683416:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078233.683424:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078233.683446:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617488000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078233.683449:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617488128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078233.683459:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078233.683461:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078233.683462:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078233.683465:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5222 00000100:00080000:2.0:1769078233.683469:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078233.683472:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078233.683476:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078233.683489:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617488256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078233.683498:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078233.683501:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078233.683503:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078233.683510:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5222 00000800:00000100:0.0:1769078233.683511:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769078233.683517:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078233.683523:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078233.683528:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078233.683541:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617488384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078233.683550:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078233/real 1769078233] req@ffff9b03482c4380 x1855007617488000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078233.683550:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078233.683556:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078233.683558:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078233.683562:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5222 00000100:00080000:2.0:1769078233.683567:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078233.683572:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078233.683577:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078233.683579:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617488000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078233.683589:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617488512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078233.683592:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078233.683601:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078233.683606:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617488000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078233.683625:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078233.683641:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078233.683651:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078233/real 1769078233] req@ffff9b03482c5180 x1855007617488128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078233.683672:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617488128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078233.683680:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078233.683683:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078233.683686:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078233.683692:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078233.683708:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078233.683715:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078233/real 1769078233] req@ffff9b03482c4a80 x1855007617488256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078233.683722:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617488256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078233.683726:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078233.683728:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078233.683730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078233.683734:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078233.683746:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078233.683751:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078233/real 1769078233] req@ffff9b03482c5500 x1855007617488384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078233.683758:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617488384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078233.683762:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078233.683764:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078233.683766:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078233.683769:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078233.683776:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078233.683781:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078233/real 1769078233] req@ffff9b03482c6300 x1855007617488512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078233.683787:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617488512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078233.683791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078233.683792:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078233.683793:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078234.195427:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:1.0:1769078234.579417:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078234.579480:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078234.579520:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078235.219356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078235.603322:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078235.603342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078235.603403:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078236.243354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078236.627343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078236.627374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078236.627406:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078237.267336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078237.651376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078237.651376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078237.651442:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078238.291297:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078238.675346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078238.675346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078238.675371:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078238.803391:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078238.803399:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078238.803402:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078238.803410:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5227 00000100:00080000:3.0:1769078238.803419:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078238.803424:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078238.803455:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617488640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078238.803473:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078238.803476:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078238.803478:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078238.803482:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5227 00000100:00080000:3.0:1769078238.803486:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078238.803490:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078238.803499:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078238.803512:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617488768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078238.803517:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617488640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078238.803535:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078238.803538:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078238.803540:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078238.803543:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5227 00000100:00080000:3.0:1769078238.803546:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078238.803550:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078238.803554:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078238.803562:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617488896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078238.803568:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078238.803570:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078238.803573:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1769078238.803574:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078238.803586:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5227 00000100:00080000:3.0:1769078238.803591:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078238.803594:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078238.803600:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078238.803613:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078238/real 1769078238] req@ffff9b0352a1a680 x1855007617488640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078238.803627:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617489024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078238.803629:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617488640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078238.803643:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078238.803655:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769078238.803655:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078238.803661:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769078238.803663:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617488640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078238.803664:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078238.803671:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5227 00000100:00080000:3.0:1769078238.803675:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078238.803677:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078238.803681:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078238.803691:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078238.803694:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617489152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078238.803727:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078238.803739:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078238/real 1769078238] req@ffff9b0352a1b480 x1855007617488768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078238.803751:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617488768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078238.803759:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078238.803762:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078238.803765:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078238.803771:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078238.803790:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078238.803800:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078238/real 1769078238] req@ffff9b0352a1ad80 x1855007617488896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078238.803817:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617488896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078238.803825:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078238.803828:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078238.803831:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617488896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078238.803837:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078238.803854:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078238.803864:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078238/real 1769078238] req@ffff9b0352a1b100 x1855007617489024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078238.803875:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617489024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078238.803883:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078238.803886:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078238.803915:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078238.803931:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078238.803950:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078238.803958:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078238/real 1769078238] req@ffff9b0352a18a80 x1855007617489152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078238.803965:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617489152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078238.803974:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078238.803976:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078238.803978:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078239.315328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078239.699336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078239.699339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078239.699385:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078240.339398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078240.723359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078240.723359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078240.723523:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078241.363329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078241.747339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078241.747339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078241.747401:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078242.387313:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078242.771359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078242.771359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078242.771430:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078243.411363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078243.795320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078243.795340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078243.795398:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078243.923340:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078243.923347:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078243.923351:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078243.923360:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5233 00000100:00080000:3.0:1769078243.923370:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078243.923375:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078243.923405:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617489280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078243.923431:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078243.923435:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078243.923437:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078243.923441:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5233 00000100:00080000:3.0:1769078243.923445:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078243.923449:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078243.923457:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078243.923467:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617489408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078243.923474:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078243.923476:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078243.923478:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078243.923485:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617489280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078243.923491:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5233 00000100:00080000:3.0:1769078243.923496:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078243.923500:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078243.923505:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078243.923517:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617489536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078243.923527:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078243.923530:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078243.923532:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078243.923536:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5233 00000100:00080000:3.0:1769078243.923540:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078243.923544:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078243.923549:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078243.923559:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617489664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078243.923577:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078243.923581:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1769078243.923584:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489408:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769078243.923584:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078243.923594:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5233 00000100:00080000:3.0:1769078243.923597:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078243.923601:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078243.923607:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078243.923618:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078243.923623:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617489792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078243.923637:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078243.923654:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078244.435315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078244.819342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078244.819342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078244.819400:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078245.459283:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078245.843347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078245.843361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078245.843380:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078246.483283:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078246.867346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078246.867346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078246.867392:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1769078246.995473:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769078246.995511:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769078246.995524:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769078246.995562:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769078246.995580:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769078246.995588:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769078246.995595:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769078246.995722:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078243/real 1769078246] req@ffff9b0352a18700 x1855007617489792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078246.995747:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617489792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078246.995771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078246.995776:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078246.995780:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078246.995790:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078243/real 1769078246] req@ffff9b0352a1a300 x1855007617489536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078246.995811:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617489536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078246.995819:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078246.995822:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078246.995826:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078246.995832:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078243/real 1769078246] req@ffff9b0352a19f80 x1855007617489664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078246.995842:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617489664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078246.995849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078246.995853:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078246.995856:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078246.995862:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078243/real 1769078246] req@ffff9b0352a1aa00 x1855007617489280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078246.995877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617489280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078246.995885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078246.995888:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078246.995891:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617489280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078246.995897:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078243/real 1769078246] req@ffff9b0352a18e00 x1855007617489408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078246.995910:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617489408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078246.995919:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078246.995924:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078246.995927:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617489408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078247.507276:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078247.891307:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078247.891307:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078247.891354:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078248.533555:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078248.915329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078248.915329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078248.915382:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078249.043335:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078249.043342:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078249.043346:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078249.043354:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5238 00000100:00080000:1.0:1769078249.043360:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078249.043366:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078249.043393:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617489920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078249.043416:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078249.043419:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078249.043421:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078249.043426:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5238 00000100:00080000:1.0:1769078249.043429:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078249.043433:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078249.043443:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078249.043455:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617490048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078249.043463:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617489920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078249.043468:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078249.043473:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078249.043475:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078249.043489:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5238 00000100:00080000:1.0:1769078249.043497:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078249.043502:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078249.043506:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078249.043518:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617490176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078249.043528:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078249.043532:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078249.043534:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078249.043540:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5238 00000800:00000100:3.0:1769078249.043540:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078249.043547:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078249.043552:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078249.043558:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078249.043569:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617490304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078249.043576:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078249.043579:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078249.043580:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769078249.043584:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078249] req@ffff9b0343dec700 x1855007617489920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078249.043587:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5238 00000100:00080000:1.0:1769078249.043594:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078249.043600:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078249.043606:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078249.043611:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617489920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078249.043619:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617490432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078249.043624:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078249.043633:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078249.043639:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617489920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078249.043664:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078249.043689:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078249.043703:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078249] req@ffff9b0343dee300 x1855007617490048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078249.043730:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617490048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078249.043740:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078249.043744:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078249.043747:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078249.043753:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078249.043774:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078249.043784:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078249] req@ffff9b0343ded180 x1855007617490176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078249.043795:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617490176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078249.043803:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078249.043806:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078249.043809:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078249.043815:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078249.043834:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078249.043844:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078249] req@ffff9b0343dec380 x1855007617490304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078249.043856:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617490304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078249.043863:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078249.043866:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078249.043868:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078249.043874:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078249.043891:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078249.043899:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078249/real 1769078249] req@ffff9b0343dec000 x1855007617490432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078249.043909:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617490432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078249.043916:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078249.043918:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078249.043921:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078249.555282:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078249.939302:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078249.939305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078249.939348:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078250.579346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078250.963355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078250.963362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078250.963406:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078251.603343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078251.987352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078251.987356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078251.987397:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078252.627337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078253.011327:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078253.011327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078253.011371:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078253.651389:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078254.035352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078254.035352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078254.035455:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078254.163446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078254.163457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078254.163461:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078254.163470:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5243 00000100:00080000:2.0:1769078254.163482:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078254.163488:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078254.163532:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617490560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078254.163569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078254.163573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078254.163575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078254.163580:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5243 00000100:00080000:2.0:1769078254.163584:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078254.163589:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078254.163598:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078254.163613:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617490688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078254.163613:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617490560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769078254.163643:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078254.163647:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078254.163649:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078254.163659:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5243 00000100:00080000:2.0:1769078254.163663:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078254.163667:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078254.163672:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078254.163682:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617490816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078254.163688:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078254.163691:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078254.163692:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078254.163696:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5243 00000100:00080000:2.0:1769078254.163706:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769078254.163707:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769078254.163711:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078254.163721:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078254.163733:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617490944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078254.163742:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078254.163745:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078254.163748:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769078254.163750:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078254/real 1769078254] req@ffff9b03482c6d80 x1855007617490560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078254.163754:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5243 00000100:00080000:2.0:1769078254.163761:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078254.163768:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078254.163774:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078254.163779:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617490560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078254.163789:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617491072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078254.163792:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078254.163801:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078254.163808:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617490560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078254.163832:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078254.163854:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078254.163865:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078254/real 1769078254] req@ffff9b03482c6680 x1855007617490688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078254.163878:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617490688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078254.163886:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078254.163890:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078254.163893:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078254.163899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078254.163918:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078254.163928:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078254/real 1769078254] req@ffff9b03482c7100 x1855007617490816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078254.163945:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617490816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078254.163952:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078254.163955:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078254.163958:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078254.163964:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078254.163982:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078254.163993:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078254/real 1769078254] req@ffff9b03482c4000 x1855007617490944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078254.164004:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617490944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078254.164012:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078254.164015:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078254.164018:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617490944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078254.164023:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078254.164039:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078254.164049:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078254/real 1769078254] req@ffff9b03482c6a00 x1855007617491072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078254.164060:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617491072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078254.164073:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078254.164076:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078254.164079:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078254.675350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078255.059368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078255.059370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078255.059434:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078255.699348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078256.083334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078256.083334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078256.083401:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078256.723336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078257.107331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078257.107331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078257.107368:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078257.747347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078258.131349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078258.131416:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078258.131445:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078258.771347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078259.155335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078259.155335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078259.155395:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078259.283389:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078259.283397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078259.283401:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078259.283410:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5248 00000100:00080000:3.0:1769078259.283417:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078259.283423:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078259.283457:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617491200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078259.283482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078259.283485:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078259.283487:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078259.283491:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5248 00000100:00080000:3.0:1769078259.283494:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078259.283499:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078259.283509:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078259.283520:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617491200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078259.283524:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617491328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078259.283537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078259.283541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078259.283555:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078259.283558:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5248 00000100:00080000:3.0:1769078259.283561:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078259.283564:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078259.283569:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078259.283582:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617491456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078259.283591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078259.283594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078259.283596:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078259.283599:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5248 00000800:00000100:0.0:1769078259.283605:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078259.283605:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078259.283613:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078259.283618:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078259.283628:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617491584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078259.283634:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078259.283636:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078259.283638:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078259.283641:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5248 00000100:00080000:3.0:1769078259.283644:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078259.283648:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078259/real 1769078259] req@ffff9b0352a18e00 x1855007617491200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078259.283649:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078259.283655:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078259.283665:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617491712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078259.283684:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617491200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078259.283695:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078259.283699:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078259.283702:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617491200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078259.283725:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078259.283747:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078259.283758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078259/real 1769078259] req@ffff9b0352a1aa00 x1855007617491328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078259.283770:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617491328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078259.283783:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078259.283786:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078259.283789:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078259.283795:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078259.283812:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078259.283822:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078259/real 1769078259] req@ffff9b0352a19f80 x1855007617491456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078259.283832:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617491456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078259.283840:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078259.283843:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078259.283846:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078259.283851:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078259.283870:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078259.283880:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078259/real 1769078259] req@ffff9b0352a1a300 x1855007617491584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078259.283890:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617491584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078259.283897:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078259.283900:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078259.283902:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078259.283907:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078259.283922:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078259.283931:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078259/real 1769078259] req@ffff9b0352a18700 x1855007617491712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078259.283942:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617491712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078259.283948:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078259.283951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078259.283954:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078259.795330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078260.179340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078260.179396:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078260.179423:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078260.819353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078261.203342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078261.203400:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078261.203443:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078261.843379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078262.227390:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078262.227663:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078262.227712:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078262.867355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078263.251383:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078263.251383:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078263.251502:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078263.891372:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078264.275346:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078264.275357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078264.275401:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078264.403348:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078264.403358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078264.403363:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078264.403371:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5253 00000100:00080000:0.0:1769078264.403379:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078264.403384:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078264.403412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617491840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078264.403435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078264.403438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078264.403440:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078264.403445:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5253 00000100:00080000:0.0:1769078264.403448:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078264.403452:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078264.403478:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078264.403494:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617491840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078264.403495:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617491968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078264.403510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078264.403512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078264.403514:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078264.403517:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5253 00000100:00080000:0.0:1769078264.403520:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078264.403523:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078264.403527:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078264.403535:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617492096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078264.403541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078264.403543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078264.403544:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078264.403547:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5253 00000100:00080000:0.0:1769078264.403551:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078264.403553:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078264.403556:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1769078264.403562:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078264.403565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617492224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078264.403572:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078264.403574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078264.403576:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078264.403579:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5253 00000100:00080000:0.0:1769078264.403582:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078264.403584:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078264.403587:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078264.403593:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617492352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078264.403600:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078264/real 1769078264] req@ffff9b0365a39180 x1855007617491840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078264.403615:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617491840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078264.403624:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078264.403628:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078264.403632:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617491840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078264.403653:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078264.403681:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078264.403694:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078264/real 1769078264] req@ffff9b0365a39500 x1855007617491968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078264.403702:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617491968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078264.403707:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078264.403710:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078264.403712:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617491968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078264.403715:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078264.403726:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078264.403732:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078264/real 1769078264] req@ffff9b0365a3bb80 x1855007617492096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078264.403738:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617492096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078264.403742:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078264.403744:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078264.403745:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078264.403749:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078264.403758:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078264.403763:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078264/real 1769078264] req@ffff9b0365a3ad80 x1855007617492224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078264.403769:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617492224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078264.403777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078264.403779:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078264.403780:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078264.403783:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078264.403792:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078264.403797:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078264/real 1769078264] req@ffff9b0365a3b100 x1855007617492352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078264.403803:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617492352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078264.403808:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078264.403810:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078264.403811:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078264.915377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078265.299351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078265.299351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078265.299404:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078265.939344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078266.323342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078266.323343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078266.323391:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078266.963343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078267.347316:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078267.347316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078267.347369:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078267.987368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078268.371378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078268.371386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078268.371446:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078269.011360:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078269.395300:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078269.395317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078269.395442:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078269.523404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078269.523411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078269.523415:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078269.523423:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5258 00000100:00080000:3.0:1769078269.523431:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078269.523437:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078269.523465:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617492480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078269.523487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078269.523490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078269.523491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078269.523494:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5258 00000100:00080000:3.0:1769078269.523496:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078269.523498:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078269.523505:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078269.523512:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617492608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078269.523520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078269.523523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078269.523524:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078269.523526:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5258 00000100:00080000:3.0:1769078269.523528:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078269.523530:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078269.523534:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078269.523541:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617492736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078269.523548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078269.523552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078269.523556:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078269.523557:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617492480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078269.523561:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5258 00000100:00080000:3.0:1769078269.523571:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078269.523576:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078269.523581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078269.523591:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617492864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078269.523600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078269.523603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078269.523604:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078269.523608:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5258 00000100:00080000:3.0:1769078269.523611:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078269.523615:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078269.523619:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078269.523630:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617492992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078269.523661:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078269.523714:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078269/real 1769078269] req@ffff9b0352a18a80 x1855007617492480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078269.523736:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617492480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078269.523748:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078269.523754:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078269.523758:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617492480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078269.523767:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078269.523792:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078269.523806:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078269/real 1769078269] req@ffff9b0352a1b100 x1855007617492608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078269.523819:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617492608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078269.523828:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078269.523831:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078269.523835:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078269.523865:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078269.523888:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078269.523899:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078269/real 1769078269] req@ffff9b0352a1ad80 x1855007617492736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078269.523911:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617492736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078269.523919:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078269.523923:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078269.523926:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078269.523933:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078269.523953:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078269.523963:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078269/real 1769078269] req@ffff9b0352a1b480 x1855007617492864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078269.523975:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617492864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078269.523999:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078269.524002:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078269.524006:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078269.524012:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078269.524032:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078269.524049:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078269/real 1769078269] req@ffff9b0352a1a680 x1855007617492992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078269.524061:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617492992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078269.524069:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078269.524073:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078269.524076:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617492992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078270.035291:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078270.419367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078270.419381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078270.419464:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078271.059337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078271.443286:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078271.443288:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078271.443382:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078272.083368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078272.467375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078272.467440:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078272.467481:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078273.107352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078273.491347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078273.491347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078273.491418:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078274.131351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078274.515361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078274.515364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078274.516342:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078274.643353:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078274.643362:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078274.643366:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078274.643375:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5263 00000100:00080000:1.0:1769078274.643382:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078274.643396:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078274.643426:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617493120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078274.643448:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078274.643451:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078274.643452:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078274.643454:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5263 00000100:00080000:1.0:1769078274.643457:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078274.643459:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078274.643465:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078274.643473:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617493248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078274.643479:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078274.643481:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078274.643482:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078274.643485:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5263 00000100:00080000:1.0:1769078274.643486:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078274.643488:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078274.643491:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078274.643497:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617493120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078274.643500:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617493376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078274.643510:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078274.643512:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078274.643513:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078274.643516:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5263 00000100:00080000:1.0:1769078274.643518:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078274.643520:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078274.643523:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078274.643531:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617493504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078274.643540:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078274.643543:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078274.643544:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078274.643548:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5263 00000100:00080000:1.0:1769078274.643552:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078274.643556:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769078274.643561:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769078274.643563:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078274.643577:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617493632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078274.643601:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078274/real 1769078274] req@ffff9b0343ded880 x1855007617493120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078274.643617:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617493120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078274.643627:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078274.643632:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078274.643635:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617493120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078274.643642:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617493248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078274.643659:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078274.643669:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078274/real 1769078274] req@ffff9b0343def800 x1855007617493248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078274.643680:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617493248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078274.643688:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078274.643691:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078274.643694:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617493248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078274.643714:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617493376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078274.643728:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078274.643736:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078274/real 1769078274] req@ffff9b0343deea00 x1855007617493376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078274.643746:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617493376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078274.643754:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078274.643757:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078274.643760:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617493376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078274.643765:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617493504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078274.643778:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078274.643785:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078274/real 1769078274] req@ffff9b0343def100 x1855007617493504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078274.643796:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617493504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078274.643803:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078274.643806:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078274.643809:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617493504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078274.643814:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617493632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078274.643826:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078274.643834:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078274/real 1769078274] req@ffff9b0343dedf80 x1855007617493632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078274.643844:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617493632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078274.643851:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078274.643854:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078274.643857:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617493632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078275.155353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078275.539374:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078275.539421:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078275.539432:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078276.179657:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078276.563351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078276.563385:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078276.564210:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078277.203365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078277.587349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078277.587349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078277.587433:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078278.227341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078278.611350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078278.612153:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078278.612411:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078279.251334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078279.635326:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078279.635326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078279.635419:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078279.763366:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078279.763377:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078279.763381:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078279.763391:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5268 00000100:00080000:3.0:1769078279.763400:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078279.763407:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078279.763447:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617493760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078279.763478:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078279.763482:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078279.763484:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078279.763489:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5268 00000100:00080000:3.0:1769078279.763493:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078279.763499:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078279.763508:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078279.763520:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617493888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078279.763530:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078279.763536:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1769078279.763538:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617493760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078279.763539:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078279.763550:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5268 00000100:00080000:3.0:1769078279.763557:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078279.763560:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078279.763563:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078279.763573:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617494016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078279.763580:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078279.763583:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078279.763584:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078279.763586:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5268 00000100:00080000:3.0:1769078279.763588:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078279.763591:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078279.763595:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078279.763600:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617494144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078279.763608:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078279.763611:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078279.763612:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078279.763615:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5268 00000100:00080000:3.0:1769078279.763620:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078279.763623:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078279.763624:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078279.763632:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078279.763640:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617494272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078279.763681:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078279/real 1769078279] req@ffff9b0352a18000 x1855007617493760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078279.763700:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617493760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078279.763709:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078279.763712:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078279.763715:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617493760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078279.763733:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617493888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078279.763750:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078279.763758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078279/real 1769078279] req@ffff9b0352a18380 x1855007617493888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078279.763765:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617493888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078279.763770:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078279.763773:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078279.763774:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617493888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078279.763778:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078279.763788:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078279.763793:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078279/real 1769078279] req@ffff9b0352a19c00 x1855007617494016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078279.763799:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617494016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078279.763804:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078279.763806:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078279.763807:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078279.763811:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078279.763820:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078279.763825:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078279/real 1769078279] req@ffff9b0352a1b800 x1855007617494144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078279.763831:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617494144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078279.763836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078279.763837:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078279.763839:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078279.763842:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078279.763854:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078279.763863:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078279/real 1769078279] req@ffff9b0352a19500 x1855007617494272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078279.763872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617494272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078279.763880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078279.763883:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078279.763886:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078280.275326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078280.659331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078280.659352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078280.659403:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078281.299329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078281.683334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078281.683334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078281.683397:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078282.323345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078282.707331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078282.707331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078282.707422:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078283.351842:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078283.731375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078283.731581:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078283.731644:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078284.371460:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078284.755352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078284.755548:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078284.755895:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769078284.883387:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078284.883398:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078284.883403:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078284.883414:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5274 00000100:00080000:3.0:1769078284.883424:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078284.883430:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078284.883470:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617494400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078284.883501:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078284.883504:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078284.883508:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078284.883511:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5274 00000100:00080000:3.0:1769078284.883515:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078284.883520:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078284.883529:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078284.883549:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617494400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078284.883549:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617494528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078284.883574:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078284.883588:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078284.883590:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078284.883598:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5274 00000100:00080000:3.0:1769078284.883603:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078284.883607:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078284.883612:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078284.883623:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617494656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078284.883633:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078284.883636:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078284.883637:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078284.883641:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5274 00000800:00000100:1.0:1769078284.883649:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078284.883698:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078284/real 1769078284] req@ffff9b0352a19500 x1855007617494400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078284.883704:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078284.883716:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078284.883724:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078284.883729:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617494400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078284.883749:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078284.883754:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078284.883759:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617494400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078284.883787:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078284.883814:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078284.883828:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078284/real 1769078284] req@ffff9b0352a1b800 x1855007617494528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078284.883850:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617494784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078284.883851:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617494528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078284.883866:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078284.883878:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078284.883886:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078284.883891:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078284.883898:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494656:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769078284.883924:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1769078284.883928:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078284.883929:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078284.883941:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5274 00000100:00080000:3.0:1769078284.883948:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078284.883955:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078284.883959:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078284/real 1769078284] req@ffff9b0352a19c00 x1855007617494656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078284.883964:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078284.883988:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617494912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078284.884001:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617494656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078284.884014:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078284.884019:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078284.884023:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078284.884039:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078284.884058:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078284.884067:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078284/real 1769078284] req@ffff9b0352a18380 x1855007617494784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078284.884079:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617494784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078284.884086:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078284.884096:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078284.884099:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078284.884105:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078284.884144:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078284.884154:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078284/real 1769078284] req@ffff9b0352a18000 x1855007617494912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078284.884165:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617494912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078284.884185:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078284.884188:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078284.884191:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617494912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078285.395374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078285.779359:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078285.779359:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078285.781301:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078286.419333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078286.803320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078286.803320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078286.803376:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078287.443341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078287.827331:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078287.827331:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078287.827391:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078288.467370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078288.851298:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078288.851300:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078288.851369:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078289.491357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078289.875357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078289.875414:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078289.875453:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:3.0:1769078290.003311:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078290.003319:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078290.003324:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078290.003333:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5279 00000100:00080000:3.0:1769078290.003342:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078290.003349:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078290.003384:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617495040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078290.003417:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078290.003421:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078290.003423:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078290.003426:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5279 00000100:00080000:3.0:1769078290.003430:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078290.003434:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078290.003443:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078290.003456:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617495168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078290.003466:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078290.003468:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078290.003470:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078290.003474:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5279 00000100:00100000:1.0:1769078290.003475:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617495040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078290.003477:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078290.003482:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078290.003487:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078290.003498:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617495296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078290.003507:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078290.003510:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078290.003512:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078290.003516:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5279 00000100:00080000:3.0:1769078290.003520:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078290.003523:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078290.003529:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078290.003538:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617495424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078290.003545:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078290.003547:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078290.003549:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1769078290.003553:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078290.003555:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5279 00000100:00080000:3.0:1769078290.003559:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078290.003561:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078290.003565:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078290.003573:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617495552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078290.003598:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9b0352a1a680 x1855007617495040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078290.003617:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617495040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078290.003627:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078290.003631:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078290.003635:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617495040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078290.003657:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078290.003678:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078290.003689:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9b0352a1b480 x1855007617495168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078290.003701:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617495168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078290.003709:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078290.003712:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078290.003725:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078290.003731:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078290.003749:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078290.003759:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9b0352a1ad80 x1855007617495296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078290.003770:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617495296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078290.003777:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078290.003780:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078290.003783:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078290.003788:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078290.003806:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078290.003815:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9b0352a1b100 x1855007617495424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078290.003834:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617495424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078290.003842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078290.003845:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078290.003848:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078290.003854:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078290.003871:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078290.003881:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078290/real 1769078290] req@ffff9b0352a18a80 x1855007617495552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078290.003892:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617495552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078290.003899:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078290.003902:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078290.003905:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078290.515344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078290.899759:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078290.899922:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078290.900098:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078291.539335:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078291.923470:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078291.924119:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078291.925181:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078292.563322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078292.947323:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078292.947388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078292.947459:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078293.587332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769078293.971528:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078293.971901:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078293.972043:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078294.611356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078294.995379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078294.995381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078294.995480:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078295.123342:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078295.123349:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078295.123353:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078295.123361:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5284 00000100:00080000:3.0:1769078295.123369:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078295.123374:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078295.123403:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617495680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078295.123429:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078295.123432:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078295.123433:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078295.123436:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5284 00000100:00080000:3.0:1769078295.123438:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078295.123440:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078295.123446:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078295.123453:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617495808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078295.123459:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078295.123462:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078295.123463:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078295.123466:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5284 00000100:00080000:3.0:1769078295.123468:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078295.123470:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078295.123473:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078295.123479:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617495936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078295.123485:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617495680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078295.123486:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078295.123492:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078295.123495:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078295.123498:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5284 00000100:00080000:3.0:1769078295.123501:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078295.123503:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078295.123506:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078295.123514:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617496064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078295.123519:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078295.123521:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078295.123522:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078295.123525:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5284 00000100:00080000:3.0:1769078295.123527:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078295.123529:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078295.123531:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078295.123537:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617496192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078295.123553:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078295.123590:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078295/real 1769078295] req@ffff9b0352a18700 x1855007617495680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078295.123607:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617495680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078295.123618:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078295.123622:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078295.123626:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617495680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078295.123632:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078295.123652:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078295.123663:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078295/real 1769078295] req@ffff9b0352a1a300 x1855007617495808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078295.123675:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617495808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078295.123684:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078295.123694:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078295.123696:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078295.123716:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078295.123731:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078295.123738:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078295/real 1769078295] req@ffff9b0352a19f80 x1855007617495936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078295.123745:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617495936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078295.123753:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078295.123755:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078295.123757:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617495936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078295.123760:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078295.123770:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078295.123775:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078295/real 1769078295] req@ffff9b0352a1aa00 x1855007617496064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078295.123781:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617496064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078295.123785:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078295.123787:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078295.123788:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078295.123792:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078295.123799:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078295.123804:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078295/real 1769078295] req@ffff9b0352a18e00 x1855007617496192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078295.123810:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617496192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078295.123814:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078295.123816:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078295.123818:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078295.635351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769078296.019548:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078296.019611:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078296.019615:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078296.659312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078297.043316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078297.043321:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078297.043396:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078297.683382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078298.067372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078298.067441:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078298.067452:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078298.707330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078299.091341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078299.091341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078299.091397:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078299.731443:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078300.115325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078300.115325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078300.115359:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078300.243361:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078300.243369:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078300.243372:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078300.243380:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5289 00000100:00080000:2.0:1769078300.243388:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078300.243393:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078300.243421:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617496320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078300.243442:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078300.243445:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078300.243446:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078300.243449:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5289 00000100:00080000:2.0:1769078300.243451:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078300.243453:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078300.243465:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078300.243475:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617496448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078300.243481:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078300.243484:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078300.243485:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078300.243488:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5289 00000100:00080000:2.0:1769078300.243490:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078300.243495:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078300.243497:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617496320:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769078300.243498:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078300.243509:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617496576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078300.243515:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078300.243517:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078300.243518:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078300.243520:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5289 00000100:00080000:2.0:1769078300.243522:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078300.243524:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078300.243527:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078300.243535:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617496704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078300.243540:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078300.243542:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078300.243543:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078300.243545:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5289 00000100:00080000:2.0:1769078300.243548:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078300.243550:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769078300.243554:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:2.0:1769078300.243555:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078300.243563:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617496832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078300.243588:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078300/real 1769078300] req@ffff9b03482c4700 x1855007617496320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078300.243600:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617496320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078300.243607:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078300.243610:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078300.243613:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617496320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078300.243618:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078300.243630:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078300.243644:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078300/real 1769078300] req@ffff9b03482c4e00 x1855007617496448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078300.243652:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617496448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078300.243657:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078300.243659:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078300.243661:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496448:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078300.243677:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496576:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078300.243688:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078300.243693:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078300/real 1769078300] req@ffff9b03482c7b80 x1855007617496576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078300.243700:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617496576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078300.243704:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078300.243706:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078300.243708:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078300.243711:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078300.243719:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078300.243724:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078300/real 1769078300] req@ffff9b03482c5f80 x1855007617496704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078300.243730:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617496704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078300.243735:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078300.243738:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078300.243740:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078300.243745:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078300.243761:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078300.243771:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078300/real 1769078300] req@ffff9b03482c5c00 x1855007617496832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078300.243782:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617496832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078300.243790:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078300.243793:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078300.243795:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617496832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078300.755312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078301.139370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078301.139375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078301.139457:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078301.779334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078302.163340:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078302.163340:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078302.163400:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078302.803357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078303.187355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078303.187355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078303.187429:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078303.827410:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078304.211356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078304.211356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078304.211419:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078304.851425:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078305.235372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078305.235432:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078305.235453:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:1.0:1769078305.363361:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078305.363372:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078305.363377:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078305.363388:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5294 00000100:00080000:1.0:1769078305.363400:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078305.363409:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078305.363456:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617496960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078305.363489:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078305.363493:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078305.363495:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078305.363499:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5294 00000100:00080000:1.0:1769078305.363504:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078305.363509:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078305.363520:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078305.363538:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617497088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078305.363539:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617496960:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078305.363555:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078305.363560:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078305.363562:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078305.363566:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5294 00000100:00080000:1.0:1769078305.363571:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078305.363575:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078305.363581:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078305.363594:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617497216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078305.363603:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078305.363607:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078305.363609:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078305.363614:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5294 00000100:00080000:1.0:1769078305.363618:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078305.363621:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078305.363625:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078305.363632:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078305.363644:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617497344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078305.363654:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078305.363658:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078305.363659:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078305.363664:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5294 00000100:00080000:1.0:1769078305.363668:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078305.363673:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078305/real 1769078305] req@ffff9b0343deca80 x1855007617496960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078305.363675:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078305.363682:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078305.363697:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617497472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078305.363702:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617496960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078305.363720:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078305.363725:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078305.363730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617496960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078305.363755:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078305.363805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078305.363818:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078305/real 1769078305] req@ffff9b0343deed80 x1855007617497088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078305.363830:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617497088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078305.363838:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078305.363842:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078305.363845:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078305.363851:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078305.363870:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078305.363880:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078305/real 1769078305] req@ffff9b0343ded500 x1855007617497216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078305.363892:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617497216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078305.363900:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078305.363903:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078305.363906:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078305.363912:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078305.363931:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078305.363942:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078305/real 1769078305] req@ffff9b0343def480 x1855007617497344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078305.363952:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617497344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078305.363959:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078305.363962:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078305.363966:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078305.363977:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078305.363997:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078305.364007:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078305/real 1769078305] req@ffff9b0343dee680 x1855007617497472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078305.364019:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617497472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078305.364026:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078305.364029:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078305.364032:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078305.875397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078306.259320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078306.259321:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078306.259405:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078306.899381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078307.283398:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078307.283399:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078307.283444:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078307.923331:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078308.307297:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078308.307298:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078308.307354:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078308.947342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078309.331367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078309.331374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078309.331427:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078309.971365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078310.355352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078310.355365:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078310.355367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769078310.483358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078310.483369:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078310.483373:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078310.483383:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5299 00000100:00080000:3.0:1769078310.483391:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078310.483398:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078310.483444:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617497600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078310.483470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078310.483473:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078310.483475:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078310.483479:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5299 00000100:00080000:3.0:1769078310.483483:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078310.483488:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078310.483497:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078310.483506:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617497600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078310.483512:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617497728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078310.483525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078310.483529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078310.483531:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078310.483536:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5299 00000100:00080000:3.0:1769078310.483540:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078310.483553:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078310.483559:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078310.483573:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617497856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078310.483584:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078310.483588:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1769078310.483589:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497728:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769078310.483591:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078310.483598:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5299 00000100:00080000:3.0:1769078310.483603:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078310.483608:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078310.483614:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078310.483615:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078310.483631:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617497984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078310.483649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078310.483653:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078310.483655:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078310.483658:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497984:192.168.204.160@tcp:8:kworker.0 00000100:00080000:3.0:1769078310.483663:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5299 00000100:00080000:3.0:1769078310.483670:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078310.483675:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078310.483691:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078310.483705:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617498112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078310.483742:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617498112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078310.995296:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078311.379348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078311.379349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078311.379434:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078312.019279:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078312.403347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078312.403347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078312.403526:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078313.043297:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078313.427341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078313.427341:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078313.427433:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769078313.555610:0:2394:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769078313.555670:0:2394:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769078313.555688:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078313.555728:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078313.555737:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078313.555748:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078313.555758:0:2394:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:1.0:1769078313.555761:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078310/real 1769078313] req@ffff9b0352a1aa00 x1855007617497728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078313.555791:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617497728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078313.555804:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078313.555810:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078313.555814:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497728:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078313.555821:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078310/real 1769078313] req@ffff9b0352a19f80 x1855007617497856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078313.555840:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617497856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078313.555847:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078313.555851:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078313.555854:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078313.555860:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078310/real 1769078313] req@ffff9b0352a18e00 x1855007617497600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078313.555872:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617497600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078313.555878:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078313.555880:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078313.555884:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617497600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078313.555913:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078310/real 1769078313] req@ffff9b0352a18700 x1855007617498112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078313.555940:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617498112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078313.555949:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078313.555952:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078313.555955:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617498112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078313.555962:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078310/real 1769078313] req@ffff9b0352a1a300 x1855007617497984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078313.555973:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617497984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078313.555980:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078313.555983:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078313.555987:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617497984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078314.067286:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078314.451363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078314.451366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078314.451437:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078315.096711:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078315.475338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078315.475339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078315.475369:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078315.603358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078315.603368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078315.603372:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078315.603382:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5304 00000100:00080000:0.0:1769078315.603390:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078315.603396:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078315.603426:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617498240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078315.603449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078315.603451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078315.603453:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078315.603456:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5304 00000100:00080000:0.0:1769078315.603465:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078315.603469:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078315.603479:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078315.603492:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617498368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078315.603499:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617498240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078315.603513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078315.603518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078315.603519:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078315.603524:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5304 00000100:00080000:0.0:1769078315.603528:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078315.603531:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078315.603536:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078315.603547:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617498496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078315.603556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078315.603559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078315.603560:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078315.603564:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5304 00000100:00080000:0.0:1769078315.603569:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078315.603572:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769078315.603577:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769078315.603578:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078315.603592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617498624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078315.603619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078315.603622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078315.603624:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078315.603630:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5304 00000100:00100000:1.0:1769078315.603630:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078315/real 1769078315] req@ffff9b0365a3b100 x1855007617498240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078315.603636:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078315.603643:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078315.603649:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078315.603657:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617498240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078315.603664:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617498752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078315.603674:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078315.603681:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078315.603687:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617498240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078315.603711:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617498368:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078315.603732:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078315.603743:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078315/real 1769078315] req@ffff9b0365a3ad80 x1855007617498368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078315.603754:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617498368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078315.603763:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078315.603767:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078315.603770:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617498368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078315.603775:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617498496:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078315.603795:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078315.603804:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078315/real 1769078315] req@ffff9b0365a3bb80 x1855007617498496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078315.603816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617498496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078315.603823:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078315.603826:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078315.603831:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617498496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078315.603836:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617498624:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078315.603853:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078315.603864:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078315/real 1769078315] req@ffff9b0365a39500 x1855007617498624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078315.603873:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617498624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078315.603880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078315.603882:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078315.603885:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617498624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078315.603890:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617498752:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078315.603905:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078315.603913:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078315/real 1769078315] req@ffff9b0365a39180 x1855007617498752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078315.603923:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617498752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078315.603929:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078315.603932:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078315.603934:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617498752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078316.115299:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078316.499350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078316.499350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078316.499464:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078317.139351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078317.523350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078317.523351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078317.523392:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078318.163301:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078318.547338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078318.547370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078318.547398:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078319.187298:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078319.571349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078319.571349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078319.571422:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078320.211304:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078320.595389:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078320.595389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078320.595441:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078320.723347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078320.723354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078320.723358:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078320.723366:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5309 00000100:00080000:0.0:1769078320.723375:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078320.723380:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078320.723410:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617498880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078320.723437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078320.723441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078320.723443:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078320.723447:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5309 00000100:00080000:0.0:1769078320.723450:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078320.723454:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078320.723470:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078320.723484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617499008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078320.723484:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617498880:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078320.723518:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078320.723521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078320.723522:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078320.723527:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5309 00000100:00080000:0.0:1769078320.723531:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078320.723534:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078320.723538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078320.723548:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617499136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078320.723554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078320.723557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078320.723559:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078320.723561:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5309 00000100:00080000:0.0:1769078320.723573:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078320.723576:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078320.723580:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769078320.723585:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078320.723592:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617499264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078320.723599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078320.723601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078320.723603:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078320.723608:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5309 00000100:00080000:0.0:1769078320.723614:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078320.723617:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078320.723620:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078320.723628:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617499392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078320.723639:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078320/real 1769078320] req@ffff9b0365a3aa00 x1855007617498880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078320.723656:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617498880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078320.723665:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078320.723671:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078320.723675:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617498880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078320.723709:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078320.723741:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078320.723752:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078320/real 1769078320] req@ffff9b0365a3b480 x1855007617499008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078320.723762:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617499008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078320.723768:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078320.723773:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078320.723778:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078320.723787:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078320.723807:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078320.723817:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078320/real 1769078320] req@ffff9b0365a38e00 x1855007617499136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078320.723825:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617499136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078320.723831:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078320.723834:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078320.723836:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078320.723841:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078320.723855:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078320.723861:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078320/real 1769078320] req@ffff9b0365a39c00 x1855007617499264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078320.723868:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617499264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078320.723872:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078320.723874:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078320.723876:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078320.723880:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078320.723888:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078320.723893:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078320/real 1769078320] req@ffff9b0365a38a80 x1855007617499392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078320.723899:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617499392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078320.723904:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078320.723905:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078320.723907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078321.235362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078321.619328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078321.619341:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078321.619394:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078322.259308:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078322.643376:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078322.643377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078322.643451:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078323.283343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078323.667335:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078323.667335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078323.667420:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078324.307351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078324.691305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078324.691305:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078324.691390:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078325.331363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078325.715315:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078325.715317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078325.715371:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078325.843334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078325.843345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078325.843349:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078325.843358:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5315 00000100:00080000:0.0:1769078325.843368:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078325.843374:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078325.843407:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617499520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078325.843459:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078325.843463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078325.843464:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078325.843469:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5315 00000100:00080000:0.0:1769078325.843472:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078325.843476:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078325.843485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078325.843499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617499648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078325.843509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1769078325.843510:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617499520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078325.843515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078325.843519:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078325.843523:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5315 00000100:00080000:0.0:1769078325.843528:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078325.843533:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078325.843538:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078325.843549:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617499776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078325.843563:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769078325.843571:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078325.843574:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078325.843576:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078325.843579:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5315 00000100:00080000:0.0:1769078325.843583:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078325.843585:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078325.843589:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078325.843598:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078325/real 1769078325] req@ffff9b0365a38a80 x1855007617499520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078325.843600:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617499904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078325.843622:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078325.843626:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078325.843628:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078325.843634:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5315 00000100:00100000:1.0:1769078325.843634:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617499520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078325.843639:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078325.843647:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:1.0:1769078325.843651:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1769078325.843653:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1769078325.843658:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078325.843664:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617499520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078325.843668:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617500032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078325.843689:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078325.843707:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078325.843715:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078325/real 1769078325] req@ffff9b0365a39c00 x1855007617499648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078325.843735:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617499648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078325.843743:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078325.843746:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078325.843749:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078325.843755:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078325.843775:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078325.843784:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078325/real 1769078325] req@ffff9b0365a38e00 x1855007617499776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078325.843795:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617499776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078325.843800:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078325.843802:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078325.843804:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078325.843807:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078325.843818:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078325.843823:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078325/real 1769078325] req@ffff9b0365a3b480 x1855007617499904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078325.843830:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617499904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078325.843834:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078325.843836:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078325.843837:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617499904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078325.843840:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078325.843848:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078325.843853:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078325/real 1769078325] req@ffff9b0365a3aa00 x1855007617500032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078325.843859:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617500032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078325.843867:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078325.843869:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078325.843871:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078326.355349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078326.739328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078326.739338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078326.739380:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078327.379394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078327.763309:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078327.763313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078327.763408:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078328.403352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078328.787339:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078328.787362:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078328.787408:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078329.427380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078329.811414:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078329.811415:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078329.811493:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078330.451370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078330.835380:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078330.835380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078330.835453:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078330.963388:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078330.963399:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078330.963404:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078330.963417:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5320 00000100:00080000:0.0:1769078330.963428:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078330.963435:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078330.963478:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617500160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078330.963501:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078330.963503:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078330.963505:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078330.963510:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5320 00000100:00080000:0.0:1769078330.963512:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078330.963515:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078330.963523:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078330.963528:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617500160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078330.963534:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617500288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078330.963542:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078330.963545:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078330.963546:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078330.963548:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5320 00000100:00080000:0.0:1769078330.963550:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078330.963552:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078330.963555:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078330.963564:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617500416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078330.963570:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078330.963573:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078330.963574:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078330.963576:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5320 00000100:00080000:0.0:1769078330.963588:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078330.963590:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078330.963594:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769078330.963599:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078330.963603:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617500544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078330.963610:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078330.963612:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078330.963614:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078330.963617:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5320 00000100:00080000:0.0:1769078330.963621:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078330.963623:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078330.963625:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078330.963634:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617500672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078330.963645:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078330/real 1769078330] req@ffff9b0365a39180 x1855007617500160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078330.963664:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617500160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078330.963676:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078330.963681:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078330.963685:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617500160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078330.963713:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078330.963734:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078330.963746:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078330/real 1769078330] req@ffff9b0365a39500 x1855007617500288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078330.963762:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617500288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078330.963771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078330.963774:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078330.963777:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078330.963782:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078330.963798:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078330.963807:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078330/real 1769078330] req@ffff9b0365a3bb80 x1855007617500416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078330.963817:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617500416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078330.963823:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078330.963827:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078330.963829:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078330.963835:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078330.963849:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078330.963857:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078330/real 1769078330] req@ffff9b0365a3ad80 x1855007617500544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078330.963867:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617500544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078330.963880:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078330.963883:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078330.963886:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078330.963892:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078330.963907:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078330.963916:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078330/real 1769078330] req@ffff9b0365a3b100 x1855007617500672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078330.963926:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617500672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078330.963933:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078330.963936:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078330.963939:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078331.475403:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078331.859361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078331.859363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078331.859424:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078332.499344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078332.883376:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078332.883381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078332.883454:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078333.523341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078333.907410:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078333.907410:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078333.907456:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078334.547344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078334.931360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078334.931366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078334.931424:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078335.571321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078335.955373:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078335.955403:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078335.955437:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078336.083327:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078336.083336:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078336.083339:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078336.083347:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5325 00000100:00080000:2.0:1769078336.083355:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078336.083367:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078336.083398:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617500800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078336.083418:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078336.083420:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078336.083422:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078336.083424:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5325 00000100:00080000:2.0:1769078336.083426:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078336.083428:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078336.083435:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078336.083444:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617500928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078336.083451:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078336.083453:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078336.083454:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078336.083457:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5325 00000100:00080000:2.0:1769078336.083459:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078336.083461:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078336.083464:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078336.083473:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617500800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078336.083482:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617501056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078336.083490:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078336.083492:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078336.083493:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078336.083496:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5325 00000100:00080000:2.0:1769078336.083498:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078336.083501:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078336.083504:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078336.083510:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617501184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078336.083516:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078336.083518:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078336.083519:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078336.083521:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5325 00000100:00080000:2.0:1769078336.083523:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078336.083525:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078336.083528:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078336.083534:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617501312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078336.083570:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078336.083623:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078336/real 1769078336] req@ffff9b03482c5880 x1855007617500800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078336.083664:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617500800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078336.083675:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078336.083680:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078336.083684:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617500800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078336.083692:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078336.083711:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078336.083720:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078336/real 1769078336] req@ffff9b03482c5c00 x1855007617500928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078336.083728:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617500928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078336.083736:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078336.083740:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078336.083743:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617500928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078336.083779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078336.083799:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078336.083809:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078336/real 1769078336] req@ffff9b03482c5f80 x1855007617501056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078336.083816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617501056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078336.083822:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078336.083824:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078336.083826:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078336.083831:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078336.083850:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078336.083860:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078336/real 1769078336] req@ffff9b03482c7b80 x1855007617501184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078336.083871:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617501184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078336.083879:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078336.083882:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078336.083886:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078336.083892:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078336.083911:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078336.083921:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078336/real 1769078336] req@ffff9b03482c4e00 x1855007617501312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078336.083932:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617501312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078336.083960:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078336.083963:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078336.083967:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078336.595362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078336.979348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078336.979366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078336.979403:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078337.619302:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078338.003374:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078338.003433:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078338.003483:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078338.643330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078339.027388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078339.027388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078339.027441:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078339.667323:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078340.051320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078340.051322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078340.051403:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078340.691342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078341.075349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078341.075361:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078341.075418:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078341.203341:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078341.203350:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078341.203353:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078341.203361:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5330 00000100:00080000:2.0:1769078341.203369:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078341.203374:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078341.203402:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617501440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078341.203429:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078341.203432:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078341.203433:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078341.203436:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5330 00000100:00080000:2.0:1769078341.203438:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078341.203440:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078341.203446:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078341.203453:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617501568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078341.203459:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078341.203462:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078341.203463:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078341.203466:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5330 00000100:00080000:2.0:1769078341.203468:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078341.203471:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078341.203475:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078341.203482:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617501696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078341.203487:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078341.203489:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078341.203490:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078341.203492:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5330 00000100:00080000:2.0:1769078341.203496:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078341.203498:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078341.203505:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078341.203514:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617501824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078341.203515:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617501440:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769078341.203535:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078341.203537:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078341.203538:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078341.203541:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5330 00000100:00080000:2.0:1769078341.203543:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078341.203546:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078341.203551:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078341.203563:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617501952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078341.203603:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078341.203648:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078341/real 1769078341] req@ffff9b03482c4700 x1855007617501440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078341.203663:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617501440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078341.203672:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078341.203675:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078341.203678:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617501440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078341.203684:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078341.203699:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078341.203706:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078341/real 1769078341] req@ffff9b03482c6a00 x1855007617501568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078341.203715:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617501568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078341.203723:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078341.203727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078341.203730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078341.203735:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078341.203754:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078341.203765:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078341/real 1769078341] req@ffff9b03482c4000 x1855007617501696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078341.203776:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617501696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078341.203781:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078341.203783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078341.203785:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078341.203810:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078341.203824:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078341.203831:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078341/real 1769078341] req@ffff9b03482c7100 x1855007617501824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078341.203838:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617501824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078341.203842:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078341.203844:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078341.203845:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078341.203849:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078341.203858:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078341.203863:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078341/real 1769078341] req@ffff9b03482c6680 x1855007617501952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078341.203879:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617501952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078341.203888:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078341.203890:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078341.203892:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617501952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078341.715361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078342.099351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078342.099351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078342.099376:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078342.739350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078343.123320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078343.123320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078343.123370:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078343.763375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078344.147345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078344.147345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078344.147402:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078344.787382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078345.171343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078345.171350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078345.171417:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078345.811380:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078346.195357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078346.195357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078346.195446:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078346.323319:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078346.323327:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078346.323331:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078346.323339:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5335 00000100:00080000:2.0:1769078346.323347:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078346.323353:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078346.323382:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617502080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078346.323399:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078346.323402:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078346.323403:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078346.323405:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5335 00000100:00080000:2.0:1769078346.323407:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078346.323409:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078346.323415:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078346.323423:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617502208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078346.323429:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078346.323432:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078346.323434:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1769078346.323439:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617502080:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769078346.323439:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5335 00000100:00080000:2.0:1769078346.323442:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078346.323448:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078346.323451:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078346.323458:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617502336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078346.323463:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078346.323465:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078346.323466:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078346.323468:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5335 00000100:00080000:2.0:1769078346.323470:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078346.323472:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078346.323475:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078346.323480:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617502464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078346.323485:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078346.323487:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078346.323488:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078346.323490:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5335 00000100:00080000:2.0:1769078346.323492:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078346.323494:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078346.323497:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078346.323504:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617502592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1769078346.323506:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078346.323547:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078346/real 1769078346] req@ffff9b03482c6d80 x1855007617502080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078346.323565:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617502080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078346.323575:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078346.323580:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078346.323584:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617502080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078346.323605:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502208:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078346.323625:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078346.323636:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078346/real 1769078346] req@ffff9b03482c6300 x1855007617502208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078346.323648:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617502208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078346.323656:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078346.323659:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078346.323662:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078346.323669:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078346.323687:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078346.323697:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078346/real 1769078346] req@ffff9b03482c5500 x1855007617502336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078346.323708:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617502336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078346.323716:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078346.323719:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078346.323722:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078346.323728:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078346.323747:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078346.323757:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078346/real 1769078346] req@ffff9b03482c4a80 x1855007617502464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078346.323768:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617502464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078346.323776:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078346.323779:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078346.323782:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078346.323788:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078346.323804:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078346.323813:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078346/real 1769078346] req@ffff9b03482c5180 x1855007617502592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078346.323825:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617502592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078346.323832:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078346.323835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078346.323838:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078346.835334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078347.219306:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078347.219306:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078347.219362:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078347.859386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078348.243375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078348.243415:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078348.244164:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078348.883368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078349.267371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078349.267443:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078349.268144:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078349.907351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078350.291391:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078350.291475:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078350.291814:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078350.931462:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078351.315337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078351.315337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078351.315394:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078351.443379:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078351.443388:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078351.443393:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078351.443402:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5340 00000100:00080000:3.0:1769078351.443411:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078351.443425:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078351.443456:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617502720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078351.443491:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078351.443495:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078351.443497:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078351.443501:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5340 00000100:00080000:3.0:1769078351.443510:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078351.443514:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078351.443522:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078351.443534:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617502848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078351.443544:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617502720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078351.443546:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078351.443553:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078351.443555:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078351.443560:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5340 00000100:00080000:3.0:1769078351.443564:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078351.443569:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078351.443573:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078351.443585:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617502976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078351.443594:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078351.443597:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078351.443599:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078351.443602:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5340 00000100:00080000:3.0:1769078351.443606:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078351.443611:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:1.0:1769078351.443612:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769078351.443619:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078351.443632:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617503104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078351.443642:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078351.443645:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078351.443647:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078351.443651:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5340 00000100:00100000:1.0:1769078351.443656:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078351/real 1769078351] req@ffff9b0352a18a80 x1855007617502720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078351.443657:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078351.443664:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078351.443671:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078351.443681:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617503232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078351.443686:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617502720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078351.443699:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078351.443704:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078351.443709:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617502720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078351.443732:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078351.443753:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078351.443771:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078351/real 1769078351] req@ffff9b0352a1b100 x1855007617502848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078351.443783:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617502848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078351.443791:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078351.443794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078351.443797:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078351.443803:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078351.443820:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078351.443830:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078351/real 1769078351] req@ffff9b0352a1ad80 x1855007617502976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078351.443841:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617502976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078351.443849:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078351.443851:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078351.443854:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617502976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078351.443860:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078351.443877:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078351.443887:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078351/real 1769078351] req@ffff9b0352a1b480 x1855007617503104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078351.443898:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617503104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078351.443904:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078351.443908:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078351.443910:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078351.443915:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078351.443930:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078351.443940:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078351/real 1769078351] req@ffff9b0352a1a680 x1855007617503232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078351.443951:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617503232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078351.443957:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078351.443960:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078351.443963:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078351.955357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078352.339366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078352.339377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078352.339415:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078352.979325:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078353.363349:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078353.363349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078353.363537:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078354.003340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078354.387337:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078354.387337:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078354.387388:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078355.027336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078355.411334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078355.411342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078355.411371:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078356.051347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078356.435345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078356.435354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078356.435408:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078356.563340:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078356.563350:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078356.563355:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078356.563364:0:69877:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5345 00000100:00080000:3.0:1769078356.563373:0:69877:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078356.563381:0:69877:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078356.563412:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617503360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078356.563439:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078356.563443:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078356.563445:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078356.563449:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5345 00000100:00080000:3.0:1769078356.563453:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078356.563464:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078356.563477:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078356.563478:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617503360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078356.563491:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617503488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078356.563503:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078356.563507:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078356.563508:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078356.563512:0:69877:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5345 00000100:00080000:3.0:1769078356.563516:0:69877:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078356.563520:0:69877:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078356.563524:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078356.563533:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617503616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078356.563540:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078356.563543:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1769078356.563547:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078356.563547:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078356.563553:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5345 00000100:00080000:3.0:1769078356.563557:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078356.563560:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078356.563565:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078356.563578:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078356/real 1769078356] req@ffff9b0352a18000 x1855007617503360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078356.563591:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617503360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078356.563597:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078356.563600:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078356.563602:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617503360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078356.563619:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078356.563634:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617503744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078356.563637:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078356.563649:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078356/real 1769078356] req@ffff9b0352a18380 x1855007617503488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078356.563649:0:69877:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078356.563657:0:69877:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078356.563662:0:69877:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078356.563666:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617503488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078356.563669:0:69877:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5345 00000100:00080000:3.0:1769078356.563676:0:69877:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078356.563677:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078356.563683:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769078356.563684:0:69877:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078356.563688:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078356.563693:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078356.563702:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078356.563707:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078356/real 1769078356] req@ffff9b0352a19c00 x1855007617503616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078356.563714:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617503616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078356.563719:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078356.563721:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078356.563723:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078356.563732:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078356.563751:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078356.563762:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078356/real 1769078356] req@ffff9b0352a1b800 x1855007617503744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078356.563774:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617503744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1769078356.563787:0:69877:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769078356.563788:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078356.563796:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078356.563800:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078356.563805:0:69877:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617503872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078356.563874:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078356.563900:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078356.563932:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078356/real 1769078356] req@ffff9b0352a19500 x1855007617503872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078356.563944:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617503872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078356.563949:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078356.563951:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078356.563953:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617503872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078357.075324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078357.459545:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078357.460212:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078357.460708:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078358.099354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078358.483339:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078358.483526:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078358.483592:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078359.123321:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078359.507402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078359.507407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078359.507451:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078360.147376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078360.531527:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078360.534034:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078360.535251:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078361.171365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769078361.555789:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078361.555806:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078361.556151:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:0.0:1769078361.683731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078361.683747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078361.683753:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078361.683768:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5350 00000100:00080000:0.0:1769078361.683782:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078361.683790:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078361.683867:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617504000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078361.683897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078361.683901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078361.683903:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078361.683908:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5350 00000100:00080000:0.0:1769078361.683912:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078361.683917:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078361.683929:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078361.683942:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617504000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078361.683947:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617504128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078361.683963:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078361.683967:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078361.683970:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078361.683976:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5350 00000100:00080000:0.0:1769078361.683979:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078361.683984:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078361.683989:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078361.684001:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617504256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078361.684017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078361.684021:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1769078361.684028:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769078361.684029:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078361.684038:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5350 00000100:00080000:0.0:1769078361.684053:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078361.684058:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078361.684063:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078361.684077:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617504384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078361.684085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078361.684092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1769078361.684092:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078361/real 1769078361] req@ffff9b0365a3b100 x1855007617504000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078361.684095:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078361.684106:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5350 00000100:00080000:0.0:1769078361.684113:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078361.684122:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078361.684128:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078361.684138:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617504512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078361.684149:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617504000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078361.684162:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078361.684168:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078361.684174:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617504000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078361.684197:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078361.684222:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078361.684261:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078361/real 1769078361] req@ffff9b0365a3ad80 x1855007617504128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078361.684271:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617504128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078361.684277:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078361.684279:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078361.684281:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504128:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078361.684286:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078361.684303:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078361.684313:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078361/real 1769078361] req@ffff9b0365a3bb80 x1855007617504256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078361.684325:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617504256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078361.684333:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078361.684336:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078361.684339:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078361.684345:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078361.684365:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078361.684375:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078361/real 1769078361] req@ffff9b0365a39500 x1855007617504384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078361.684382:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617504384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078361.684387:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078361.684389:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078361.684391:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078361.684395:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078361.684414:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078361.684420:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078361/real 1769078361] req@ffff9b0365a39180 x1855007617504512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078361.684426:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617504512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078361.684431:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078361.684433:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078361.684435:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078362.195351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078362.580289:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078362.580479:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078362.581971:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078363.219390:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078363.603382:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078363.603421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078363.603451:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078364.243355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078364.627342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078364.627350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078364.627412:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078365.267342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078365.651347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078365.651357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078365.652322:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078366.291347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078366.675309:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078366.675329:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078366.675401:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078366.803347:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078366.803355:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078366.803358:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078366.803367:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5355 00000100:00080000:1.0:1769078366.803374:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078366.803379:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078366.803411:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617504640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078366.803429:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078366.803432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078366.803433:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078366.803436:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5355 00000100:00080000:1.0:1769078366.803438:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078366.803440:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078366.803447:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078366.803455:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617504768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078366.803471:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617504640:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078366.803472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078366.803478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078366.803481:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078366.803484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5355 00000100:00080000:1.0:1769078366.803486:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078366.803488:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078366.803491:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078366.803501:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617504896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078366.803510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078366.803513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078366.803514:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078366.803518:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5355 00000100:00080000:1.0:1769078366.803522:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078366.803526:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078366.803530:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078366.803540:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078366.803543:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617505024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078366.803562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078366.803565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078366.803567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078366.803572:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5355 00000100:00100000:2.0:1769078366.803576:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078366/real 1769078366] req@ffff9b0343defb80 x1855007617504640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078366.803578:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078366.803586:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078366.803595:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078366.803611:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617504640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078366.803615:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617505152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078366.803630:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078366.803636:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078366.803641:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617504640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078366.803661:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078366.803680:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078366.803690:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078366/real 1769078366] req@ffff9b0343dece00 x1855007617504768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078366.803701:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617504768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078366.803709:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078366.803712:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078366.803716:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078366.803721:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078366.803748:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078366.803758:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078366/real 1769078366] req@ffff9b0343dee680 x1855007617504896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078366.803769:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617504896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078366.803782:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078366.803785:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078366.803788:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617504896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078366.803794:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078366.803809:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078366.803818:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078366/real 1769078366] req@ffff9b0343def480 x1855007617505024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078366.803828:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617505024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078366.803836:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078366.803839:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078366.803842:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078366.803847:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078366.803860:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078366.803867:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078366/real 1769078366] req@ffff9b0343ded500 x1855007617505152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078366.803877:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617505152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078366.803884:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078366.803886:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078366.803890:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078367.315341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078367.699344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078367.699363:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078367.699376:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078368.339348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078368.723334:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078368.723444:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078368.723484:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078369.363333:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078369.747332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078369.747350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078369.747500:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078370.387384:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078370.771336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078370.771336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078370.771414:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078371.411346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078371.795336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078371.795336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078371.795440:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078371.923382:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078371.923391:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078371.923395:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078371.923402:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5361 00000100:00080000:3.0:1769078371.923412:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078371.923417:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078371.923448:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617505280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078371.923469:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078371.923472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078371.923474:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078371.923478:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5361 00000100:00080000:3.0:1769078371.923481:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078371.923485:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078371.923495:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078371.923506:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617505408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078371.923515:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078371.923518:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769078371.923520:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617505280:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078371.923520:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078371.923523:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5361 00000100:00080000:3.0:1769078371.923526:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078371.923529:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078371.923533:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078371.923541:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617505536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078371.923548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078371.923551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078371.923552:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078371.923555:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5361 00000100:00080000:3.0:1769078371.923565:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078371.923568:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078371.923572:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078371.923580:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617505664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078371.923586:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078371.923589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078371.923592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078371.923593:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078371.923596:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5361 00000100:00080000:3.0:1769078371.923599:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078371.923601:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078371.923605:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078371.923612:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617505792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078371.923627:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078371/real 1769078371] req@ffff9b0352a19500 x1855007617505280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078371.923644:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617505280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078371.923679:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078371.923683:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078371.923687:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617505280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078371.923711:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078371.923734:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078371.923745:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078371/real 1769078371] req@ffff9b0352a1b800 x1855007617505408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078371.923753:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617505408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078371.923765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078371.923767:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078371.923770:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078371.923777:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078371.923794:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078371.923804:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078371/real 1769078371] req@ffff9b0352a19c00 x1855007617505536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078371.923816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617505536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078371.923833:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078371.923837:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078371.923840:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078371.923845:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078371.923862:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078371.923872:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078371/real 1769078371] req@ffff9b0352a18380 x1855007617505664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078371.923883:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617505664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078371.923900:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078371.923904:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078371.923907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078371.923913:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078371.923932:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078371.923943:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078371/real 1769078371] req@ffff9b0352a18000 x1855007617505792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078371.923961:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617505792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078371.923968:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078371.923971:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078371.923975:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617505792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078372.435434:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078372.819336:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078372.819336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078372.819366:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078373.459363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078373.843352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078373.843354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078373.843411:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078374.483322:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078374.867352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078374.867352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078374.867419:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078375.507339:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078375.891361:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078375.891440:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078375.891468:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078376.531328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078376.915378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078376.915381:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078376.915695:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078377.043352:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078377.043362:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078377.043367:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078377.043376:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5366 00000100:00080000:3.0:1769078377.043385:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078377.043392:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078377.043422:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617505920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078377.043443:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078377.043446:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078377.043447:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078377.043450:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5366 00000100:00080000:3.0:1769078377.043452:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078377.043454:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078377.043461:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078377.043470:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617506048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078377.043477:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078377.043480:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769078377.043481:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617505920:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078377.043482:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078377.043488:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5366 00000100:00080000:3.0:1769078377.043492:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078377.043494:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078377.043496:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078377.043503:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617506176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078377.043509:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078377.043511:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078377.043512:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078377.043514:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5366 00000100:00080000:3.0:1769078377.043517:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078377.043519:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078377.043523:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078377.043529:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617506304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078377.043534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078377.043536:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078377.043537:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078377.043540:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5366 00000100:00080000:3.0:1769078377.043542:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078377.043544:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078377.043547:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078377.043553:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617506432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078377.043604:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078377.043636:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078377.043770:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078377.043796:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078377.555312:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078377.939356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078377.939356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078377.939381:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078378.579513:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769078378.963433:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078378.963488:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078378.963595:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078379.603324:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078379.987350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078379.987367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078379.987447:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1769078380.115559:0:2393:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:0.0:1769078380.115611:0:2393:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1769078380.115632:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078380.115681:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078380.115690:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078380.115697:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:0.0:1769078380.115705:0:2393:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769078380.115778:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078380] req@ffff9b0352a1b480 x1855007617506048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078380.115805:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617506048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078380.115829:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078380.115835:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078380.115839:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078380.115849:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078380] req@ffff9b0352a1ad80 x1855007617506176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078380.115861:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617506176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078380.115869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078380.115872:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078380.115875:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078380.115882:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078380] req@ffff9b0352a1b100 x1855007617506304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078380.115894:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617506304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078380.115901:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078380.115905:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078380.115908:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078380.115914:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078380] req@ffff9b0352a18a80 x1855007617506432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078380.115925:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617506432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078380.115934:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078380.115937:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078380.115941:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078380.115947:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078377/real 1769078380] req@ffff9b0352a1a680 x1855007617505920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078380.115958:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617505920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078380.115965:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078380.115968:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078380.115971:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617505920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078380.627303:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078381.011367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078381.011404:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078381.011409:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078381.651351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078382.035305:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078382.035305:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078382.035356:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078382.163383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078382.163394:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078382.163398:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078382.163408:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5371 00000100:00080000:3.0:1769078382.163418:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078382.163425:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078382.163455:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617506560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078382.167618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078382.167627:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078382.167631:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078382.167642:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5371 00000100:00080000:3.0:1769078382.167654:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078382.167661:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078382.167673:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078382.167827:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617506688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078382.167842:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078382.167846:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078382.167847:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078382.167852:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5371 00000100:00080000:3.0:1769078382.167856:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078382.167860:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078382.167865:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078382.167879:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617506816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078382.167888:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078382.167891:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078382.167893:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078382.167896:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5371 00000100:00080000:3.0:1769078382.167900:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078382.167903:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078382.167908:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078382.167918:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617506944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078382.167927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078382.167930:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078382.167932:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078382.167935:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5371 00000100:00080000:3.0:1769078382.167938:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078382.167941:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078382.167945:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078382.167956:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617507072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078382.168940:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617506560:192.168.204.160@tcp:250:kworker.0 00000800:00000100:2.0:1769078382.169002:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078382.169039:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078382/real 1769078382] req@ffff9b0352a1a300 x1855007617506560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078382.169056:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617506560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078382.169066:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078382.169071:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078382.169074:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617506560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078382.169081:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506688:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078382.169102:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078382.169114:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078382/real 1769078382] req@ffff9b0352a18700 x1855007617506688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078382.169126:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617506688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078382.169134:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078382.169137:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078382.169140:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078382.169146:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506816:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078382.169165:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078382.169176:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078382/real 1769078382] req@ffff9b0352a18e00 x1855007617506816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078382.169186:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617506816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078382.169193:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078382.169196:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078382.169199:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078382.169204:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506944:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078382.169222:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078382.169257:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078382/real 1769078382] req@ffff9b0352a19f80 x1855007617506944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078382.169270:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617506944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078382.169277:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078382.169279:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078382.169282:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617506944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078382.169286:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507072:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078382.169303:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078382.169313:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078382/real 1769078382] req@ffff9b0352a1aa00 x1855007617507072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078382.169323:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617507072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078382.169329:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078382.169332:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078382.169335:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078382.675320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078383.059328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078383.059328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078383.059349:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078383.699332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078384.083320:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078384.083320:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078384.083352:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078384.723338:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078385.107345:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078385.107347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078385.107375:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078385.747296:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078386.131289:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078386.131290:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078386.131316:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078386.771347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078387.155342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078387.155348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078387.155422:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078387.283351:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078387.283362:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078387.283366:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078387.283377:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5376 00000100:00080000:0.0:1769078387.283412:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078387.283420:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078387.283454:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617507200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078387.283489:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078387.283493:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078387.283495:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078387.283500:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5376 00000100:00080000:0.0:1769078387.283503:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078387.283508:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078387.283517:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078387.283529:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617507328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078387.283542:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078387.283548:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1769078387.283548:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617507200:192.168.204.160@tcp:250:kworker.0 00000100:00080000:0.0:1769078387.283552:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078387.283564:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5376 00000100:00080000:0.0:1769078387.283568:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078387.283572:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078387.283577:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078387.283588:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617507456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078387.283599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078387.283602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078387.283605:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078387.283609:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5376 00000100:00080000:0.0:1769078387.283613:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078387.283616:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078387.283620:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078387.283630:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617507584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078387.283640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078387.283643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078387.283646:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078387.283650:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5376 00000100:00080000:0.0:1769078387.283654:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078387.283658:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078387.283663:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078387.283663:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078387.283684:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617507712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078387.283718:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078387/real 1769078387] req@ffff9b0365a3aa00 x1855007617507200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078387.283737:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617507200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078387.283746:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078387.283750:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078387.283753:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617507200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078387.283779:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078387.283803:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078387.283816:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078387/real 1769078387] req@ffff9b0365a3b480 x1855007617507328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078387.283828:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617507328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078387.283835:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078387.283838:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078387.283840:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078387.283845:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078387.283859:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078387.283868:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078387/real 1769078387] req@ffff9b0365a38e00 x1855007617507456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078387.283880:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617507456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078387.283888:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078387.283892:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078387.283895:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078387.283901:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078387.283921:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078387.283928:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078387/real 1769078387] req@ffff9b0365a39c00 x1855007617507584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078387.283936:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617507584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078387.283941:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078387.283943:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078387.283945:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078387.283949:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078387.283960:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078387.283965:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078387/real 1769078387] req@ffff9b0365a38a80 x1855007617507712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078387.283971:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617507712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078387.283980:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078387.283982:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078387.283984:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078387.795336:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078388.179343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078388.179343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078388.179364:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078388.819340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078389.203314:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078389.203314:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078389.203387:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078389.843334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078390.227429:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078390.227437:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078390.227624:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078390.867495:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078391.251414:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078391.251414:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078391.251502:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078391.891379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078392.275358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078392.275366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078392.275417:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1769078392.403360:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078392.403371:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078392.403375:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078392.403384:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5381 00000100:00080000:0.0:1769078392.403392:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078392.403405:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078392.403440:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38a80 x1855007617507840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078392.403470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078392.403474:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078392.403476:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078392.403480:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5381 00000100:00080000:0.0:1769078392.403484:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078392.403489:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078392.403499:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078392.403510:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617507840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078392.403513:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39c00 x1855007617507968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078392.403525:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078392.403529:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078392.403531:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078392.403535:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5381 00000100:00080000:0.0:1769078392.403539:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078392.403542:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078392.403547:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078392.403558:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a38e00 x1855007617508096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078392.403568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078392.403571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078392.403574:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078392.403579:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5381 00000100:00080000:0.0:1769078392.403583:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1769078392.403588:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769078392.403594:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078392.403615:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078392.403631:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b480 x1855007617508224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078392.403631:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078392/real 1769078392] req@ffff9b0365a38a80 x1855007617507840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078392.403645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078392.403654:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078392.403657:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078392.403662:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5381 00000100:00080000:0.0:1769078392.403665:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078392.403670:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078392.403674:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078392.403684:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38a80 x1855007617507840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078392.403687:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3aa00 x1855007617508352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078392.403710:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078392.403716:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078392.403720:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617507840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078392.403746:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078392.403771:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078392.403784:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078392/real 1769078392] req@ffff9b0365a39c00 x1855007617507968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078392.403796:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39c00 x1855007617507968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078392.403804:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078392.403807:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078392.403810:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617507968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078392.403815:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078392.403833:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078392.403842:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078392/real 1769078392] req@ffff9b0365a38e00 x1855007617508096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078392.403851:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a38e00 x1855007617508096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078392.403872:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078392.403875:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078392.403877:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a38e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078392.403883:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078392.403900:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078392.403910:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078392/real 1769078392] req@ffff9b0365a3b480 x1855007617508224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078392.403920:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b480 x1855007617508224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078392.403927:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078392.403930:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078392.403933:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078392.403938:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078392.403964:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078392.403975:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078392/real 1769078392] req@ffff9b0365a3aa00 x1855007617508352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078392.403992:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3aa00 x1855007617508352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078392.404004:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078392.404007:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078392.404010:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078392.915368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078393.299389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078393.299552:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078393.299756:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078393.939364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078394.323360:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078394.323360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078394.324311:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078394.963386:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078395.347354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078395.347354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078395.347412:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078395.987376:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078396.371366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078396.371490:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078396.372992:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078397.013327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078397.395313:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078397.395639:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078397.395688:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078397.523502:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078397.523512:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078397.523516:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078397.523532:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5386 00000100:00080000:3.0:1769078397.523541:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078397.523547:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078397.523582:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617508480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078397.523656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078397.523660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078397.523662:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078397.523667:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5386 00000100:00080000:3.0:1769078397.523671:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078397.523676:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078397.523686:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078397.523705:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617508608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078397.523707:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617508480:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078397.523731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078397.523734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078397.523736:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078397.523741:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5386 00000100:00080000:3.0:1769078397.523744:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078397.523748:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078397.523754:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078397.523764:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617508736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078397.523774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078397.523777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078397.523780:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078397.523785:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5386 00000100:00080000:3.0:1769078397.523789:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078397.523802:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078397.523807:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769078397.523817:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078397.523819:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617508864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078397.523831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078397.523834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078397.523837:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078397.523848:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5386 00000100:00080000:3.0:1769078397.523852:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078397.523856:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078397.523861:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078397.523873:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617508992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078397.523880:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078397/real 1769078397] req@ffff9b0352a1aa00 x1855007617508480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078397.523924:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617508480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078397.523937:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078397.523942:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078397.523947:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617508480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078397.523989:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078397.524015:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078397.524028:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078397/real 1769078397] req@ffff9b0352a19f80 x1855007617508608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078397.524054:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617508608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078397.524062:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078397.524066:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078397.524069:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508608:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078397.524076:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078397.524097:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078397.524109:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078397/real 1769078397] req@ffff9b0352a18e00 x1855007617508736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078397.524120:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617508736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078397.524128:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078397.524131:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078397.524135:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078397.524141:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078397.524159:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078397.524166:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078397/real 1769078397] req@ffff9b0352a18700 x1855007617508864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078397.524174:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617508864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078397.524183:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078397.524186:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078397.524188:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078397.524191:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078397.524205:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078397.524283:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078397/real 1769078397] req@ffff9b0352a1a300 x1855007617508992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078397.524298:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617508992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078397.524307:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078397.524310:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078397.524313:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617508992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078398.035357:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078398.419371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078398.419532:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078398.419654:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078399.059353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078399.443356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078399.443570:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078399.449268:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078400.083361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078400.467369:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078400.467470:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078400.467615:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078401.107327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078401.491328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078401.491328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078401.493298:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078402.131369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078402.515350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078402.515389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078402.515581:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078402.643394:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078402.643406:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078402.643410:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078402.643420:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5391 00000100:00080000:1.0:1769078402.643432:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078402.643439:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078402.643473:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617509120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078402.643507:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078402.643511:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078402.643513:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078402.643517:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5391 00000100:00080000:1.0:1769078402.643521:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078402.643524:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078402.643532:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078402.643542:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617509248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078402.643551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078402.643554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078402.643556:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078402.643559:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5391 00000100:00080000:1.0:1769078402.643562:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078402.643565:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078402.643568:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078402.643576:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617509376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078402.643584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078402.643586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078402.643588:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078402.643591:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5391 00000100:00080000:1.0:1769078402.643594:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078402.643597:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078402.643601:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078402.643608:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617509504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078402.643616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078402.643618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078402.643620:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078402.643622:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5391 00000100:00080000:1.0:1769078402.643625:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078402.643628:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078402.643632:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078402.643639:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617509632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078402.643763:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617509120:192.168.204.160@tcp:250:kworker.0 00000800:00000100:2.0:1769078402.643819:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078402.643851:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078402/real 1769078402] req@ffff9b0343deed80 x1855007617509120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078402.643878:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617509120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078402.643895:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078402.643898:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078402.643902:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617509120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078402.643908:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617509248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078402.643927:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078402.643937:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078402/real 1769078402] req@ffff9b0343deca80 x1855007617509248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078402.643949:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617509248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078402.643957:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078402.643959:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078402.643962:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617509248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078402.643968:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617509376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078402.643986:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078402.643995:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078402/real 1769078402] req@ffff9b0343dedf80 x1855007617509376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078402.644007:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617509376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078402.644015:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078402.644018:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078402.644021:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617509376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078402.644032:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617509504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078402.644059:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078402.644070:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078402/real 1769078402] req@ffff9b0343def100 x1855007617509504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078402.644081:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617509504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078402.644089:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078402.644092:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078402.644095:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617509504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078402.644101:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617509632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078402.644115:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078402.644123:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078402/real 1769078402] req@ffff9b0343deea00 x1855007617509632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078402.644159:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617509632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078402.644167:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078402.644170:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078402.644173:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617509632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078403.155542:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769078403.539385:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078403.543419:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078403.543753:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078404.179418:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078404.563353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078404.564492:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078404.564508:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078405.203528:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078405.587413:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078405.587657:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078405.587914:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078406.227877:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078406.613191:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078406.614280:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078406.615952:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078407.251356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078407.635328:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078407.635348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078407.635532:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078407.763898:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078407.763907:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078407.763912:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078407.763921:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5396 00000100:00080000:3.0:1769078407.763931:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078407.763937:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078407.763972:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617509760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078407.763996:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078407.764000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078407.764002:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078407.764005:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5396 00000100:00080000:3.0:1769078407.764009:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078407.764013:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078407.764022:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078407.764035:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617509888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078407.764059:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617509760:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078407.764061:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078407.764068:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078407.764072:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078407.764078:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5396 00000100:00080000:3.0:1769078407.764082:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078407.764086:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078407.764092:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078407.764103:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617510016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078407.764113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078407.764117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078407.764119:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078407.764122:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5396 00000100:00080000:3.0:1769078407.764125:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078407.764129:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769078407.764133:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:3.0:1769078407.764137:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078407.764150:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617510144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078407.764160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078407.764164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078407.764166:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078407.764170:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5396 00000100:00080000:3.0:1769078407.764175:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078407.764179:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078407/real 1769078407] req@ffff9b0352a1a680 x1855007617509760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078407.764181:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078407.764188:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078407.764204:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617510272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078407.764206:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617509760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078407.764228:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078407.764253:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078407.764258:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617509760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078407.764280:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617509888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078407.764303:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078407.764314:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078407/real 1769078407] req@ffff9b0352a18a80 x1855007617509888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078407.764326:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617509888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078407.764334:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078407.764337:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078407.764340:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617509888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078407.764347:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078407.764364:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078407.764374:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078407/real 1769078407] req@ffff9b0352a1b100 x1855007617510016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078407.764392:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617510016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078407.764400:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078407.764403:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078407.764406:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078407.764412:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078407.764430:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078407.764440:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078407/real 1769078407] req@ffff9b0352a1ad80 x1855007617510144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078407.764451:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617510144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078407.764458:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078407.764461:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078407.764464:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078407.764470:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078407.764487:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078407.764496:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078407/real 1769078407] req@ffff9b0352a1b480 x1855007617510272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078407.764507:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617510272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078407.764520:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078407.764523:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078407.764526:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078408.275351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078408.659367:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078408.659415:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078408.659455:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078409.299473:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078409.683412:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078409.683509:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078409.683543:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078410.323438:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078410.707436:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078410.707473:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078410.708468:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078411.347366:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078411.731335:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078411.731457:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078411.731526:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078412.371402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078412.755317:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078412.755377:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078412.755401:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:0.0:1769078412.883376:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078412.883395:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078412.883401:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078412.883413:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5402 00000100:00080000:0.0:1769078412.883424:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078412.883438:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078412.883484:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617510400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078412.883510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078412.883514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078412.883516:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078412.883520:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5402 00000100:00080000:0.0:1769078412.883523:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078412.883527:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078412.883537:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078412.883544:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617510400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078412.883551:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617510528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078412.883562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078412.883566:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078412.883567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078412.883571:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5402 00000100:00080000:0.0:1769078412.883574:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078412.883577:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078412.883581:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078412.883590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617510656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078412.883600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078412.883603:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078412.883605:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078412.883610:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5402 00000100:00080000:0.0:1769078412.883613:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078412.883617:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:2.0:1769078412.883621:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:0.0:1769078412.883624:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078412.883635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617510784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078412.883645:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078412.883648:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078412.883650:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078412.883653:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5402 00000100:00080000:0.0:1769078412.883656:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078412.883665:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078412.883670:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078412.883675:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078412/real 1769078412] req@ffff9b0365a39180 x1855007617510400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078412.883681:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617510912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078412.883710:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617510400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078412.883722:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078412.883726:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078412.883730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617510400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078412.883753:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078412.883770:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078412.883779:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078412/real 1769078412] req@ffff9b0365a39500 x1855007617510528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078412.883786:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617510528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078412.883792:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078412.883794:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078412.883796:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510528:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078412.883800:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078412.883809:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078412.883814:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078412/real 1769078412] req@ffff9b0365a3bb80 x1855007617510656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078412.883820:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617510656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078412.883825:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078412.883827:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078412.883828:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078412.883831:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078412.883850:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078412.883856:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078412/real 1769078412] req@ffff9b0365a3ad80 x1855007617510784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078412.883863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617510784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078412.883867:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078412.883869:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078412.883872:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078412.883875:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078412.883884:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078412.883889:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078412/real 1769078412] req@ffff9b0365a3b100 x1855007617510912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078412.883895:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617510912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078412.883900:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078412.883901:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078412.883903:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617510912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078413.395320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078413.779370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078413.779370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078413.779431:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078414.419607:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769078414.803527:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078414.803603:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078414.804034:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078415.443480:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078415.827378:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078415.827534:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078415.828022:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078416.467568:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078416.851352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078416.851378:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078416.851443:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078417.491382:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078417.875397:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078417.875412:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078417.875515:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078418.003364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078418.003380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078418.003385:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078418.003399:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5407 00000100:00080000:1.0:1769078418.003412:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078418.003419:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078418.003462:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617511040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078418.003486:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078418.003489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078418.003491:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078418.003494:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5407 00000100:00080000:1.0:1769078418.003496:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078418.003499:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078418.003508:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078418.003517:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617511168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078418.003525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078418.003530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078418.003533:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078418.003537:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5407 00000100:00080000:1.0:1769078418.003540:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078418.003542:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617511040:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078418.003552:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078418.003556:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078418.003568:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617511296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078418.003586:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078418.003590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078418.003593:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078418.003598:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5407 00000100:00080000:1.0:1769078418.003611:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1769078418.003618:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078418.003619:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078418.003628:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078418.003643:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617511424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078418.003653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078418.003656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078418.003658:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078418.003671:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9b0343def800 x1855007617511040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078418.003677:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5407 00000100:00080000:1.0:1769078418.003685:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078418.003707:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078418.003709:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617511040/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769078418.003716:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769078418.003733:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769078418.003739:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617511552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078418.003745:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078418.003763:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617511040:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078418.003786:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511168:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078418.003806:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078418.003817:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9b0343ded880 x1855007617511168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.003829:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617511168/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.003837:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.003841:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078418.003844:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511168:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078418.003850:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511296:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078418.003892:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078418.003906:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9b0343dec000 x1855007617511296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.003918:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617511296/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.003932:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.003935:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078418.003938:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511296:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078418.003944:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511424:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078418.003961:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078418.003971:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9b0343dec380 x1855007617511424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.003981:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617511424/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.003988:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.003990:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078418.003993:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511424:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078418.003999:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511552:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078418.004015:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078418.004025:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078418/real 1769078418] req@ffff9b0343ded180 x1855007617511552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078418.004035:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617511552/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078418.004042:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078418.004044:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078418.004047:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511552:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078418.515369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078418.899292:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078418.899394:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078418.900275:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078419.539369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078419.923370:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078419.923370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078419.923417:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078420.563375:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078420.947377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078420.947377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078420.947433:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078421.587385:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078421.971379:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078421.971413:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078421.971527:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078422.611401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078422.995352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078422.995352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078422.995419:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078423.123372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078423.123387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078423.123392:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078423.123404:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5412 00000100:00080000:3.0:1769078423.123416:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078423.123424:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078423.123479:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617511680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078423.123506:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078423.123510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078423.123512:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078423.123516:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5412 00000100:00080000:3.0:1769078423.123519:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078423.123523:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078423.123533:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078423.123545:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617511808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078423.123552:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617511680:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078423.123559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078423.123564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078423.123566:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078423.123571:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5412 00000100:00080000:3.0:1769078423.123575:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078423.123579:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078423.123583:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078423.123594:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617511936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078423.123603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078423.123607:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078423.123609:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078423.123613:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5412 00000100:00080000:3.0:1769078423.123636:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078423.123641:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078423.123646:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1769078423.123654:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078423.123658:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617512064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078423.123668:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078423.123672:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078423.123674:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078423.123679:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5412 00000100:00080000:3.0:1769078423.123682:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078423.123686:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078423.123692:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078423.123704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078423/real 1769078423] req@ffff9b0352a18000 x1855007617511680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078423.123706:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617512192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078423.123746:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617511680/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078423.123756:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078423.123761:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078423.123765:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617511680:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078423.123807:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511808:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078423.123830:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078423.123842:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078423/real 1769078423] req@ffff9b0352a18380 x1855007617511808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078423.123854:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617511808/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078423.123862:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078423.123865:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078423.123869:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511808:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078423.123874:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511936:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078423.123900:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078423.123910:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078423/real 1769078423] req@ffff9b0352a19c00 x1855007617511936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078423.123920:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617511936/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078423.123928:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078423.123931:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078423.123934:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617511936:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078423.123940:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512064:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078423.123957:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078423.123965:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078423/real 1769078423] req@ffff9b0352a1b800 x1855007617512064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078423.123976:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617512064/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078423.123983:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078423.123986:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078423.123990:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512064:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078423.123995:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512192:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078423.124010:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078423.124019:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078423/real 1769078423] req@ffff9b0352a19500 x1855007617512192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078423.124029:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617512192/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078423.124036:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078423.124039:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078423.124042:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512192:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078423.635398:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078424.019405:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078424.020069:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078424.024684:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078424.659586:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078425.043429:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078425.043462:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078425.045221:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078425.683362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078426.067371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078426.067679:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078426.067914:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078426.707341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769078427.091757:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078427.092832:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078427.092843:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078427.731346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769078428.115538:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078428.116743:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078428.116749:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:3.0:1769078428.243333:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078428.243341:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078428.243345:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078428.243353:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5417 00000100:00080000:3.0:1769078428.243366:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078428.243372:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078428.243404:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617512320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078428.243429:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078428.243433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078428.243435:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078428.243439:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5417 00000100:00080000:3.0:1769078428.243444:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078428.243449:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078428.243459:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078428.243461:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617512320:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078428.243475:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617512448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078428.243492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078428.243496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078428.243498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078428.243503:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5417 00000100:00080000:3.0:1769078428.243507:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078428.243512:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078428.243518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1769078428.243524:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078428.243538:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617512576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078428.243548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078428.243552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078428.243553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078428.243558:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5417 00000100:00080000:3.0:1769078428.243562:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078428.243565:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078428/real 1769078428] req@ffff9b0352a19500 x1855007617512320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078428.243570:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078428.243579:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078428.243587:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617512320/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078428.243595:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617512704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078428.243607:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078428.243612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078428.243613:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078428.243618:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617512320:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078428.243620:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078428.243624:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078428.243639:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512448:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078428.243660:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078428.243670:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078428/real 1769078428] req@ffff9b0352a1b800 x1855007617512448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078428.243682:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617512448/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078428.243691:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078428.243695:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00080000:3.0:1769078428.243717:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5417 00000100:00080000:3.0:1769078428.243723:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1769078428.243726:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512448:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769078428.243731:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078428.243738:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512576:192.168.204.160@tcp:38:kworker.0 00000008:00100000:3.0:1769078428.243739:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078428.243757:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617512832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1769078428.243766:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078428.243781:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078428/real 1769078428] req@ffff9b0352a19c00 x1855007617512576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078428.243791:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617512576/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078428.243796:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078428.243798:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078428.243800:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512576:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078428.243804:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512704:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078428.243816:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078428.243829:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078428/real 1769078428] req@ffff9b0352a18380 x1855007617512704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078428.243838:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617512704/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078428.243846:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078428.243849:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078428.243851:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512704:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078428.243868:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512832:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078428.243889:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078428.243898:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078428/real 1769078428] req@ffff9b0352a18000 x1855007617512832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078428.243909:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617512832/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078428.243918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078428.243922:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078428.243925:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617512832:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078428.755394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078429.139391:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078429.139552:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078429.139664:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078429.779407:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078430.163378:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078430.163390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078430.163719:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078430.803349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078431.187368:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078431.187370:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078431.187574:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078431.827337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078432.211322:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:3.0:1769078432.211327:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078432.211374:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078432.851615:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078433.235392:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078433.235472:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078433.235526:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769078433.363403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078433.363423:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078433.363428:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078433.363440:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5422 00000100:00080000:2.0:1769078433.363453:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078433.363462:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078433.363516:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617512960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078433.363549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078433.363552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078433.363553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078433.363557:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5422 00000100:00080000:2.0:1769078433.363560:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078433.363563:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078433.363572:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078433.363586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617513088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078433.363593:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078433.363596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078433.363597:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078433.363600:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5422 00000100:00080000:2.0:1769078433.363602:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078433.363606:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078433.363644:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617512960:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769078433.363663:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078433.363683:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617513216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078433.363696:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078433.363701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078433.363704:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078433.363708:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5422 00000100:00080000:2.0:1769078433.363714:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078433.363718:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078433.363724:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1769078433.363730:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078433.363742:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617513344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078433.363751:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078433.363754:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078433.363755:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078433.363758:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5422 00000100:00080000:2.0:1769078433.363760:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078433.363764:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078433.363768:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078433.363776:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617513472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078433.363780:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078433/real 1769078433] req@ffff9b03482c4380 x1855007617512960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078433.363809:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617512960/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078433.363817:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078433.363822:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078433.363825:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617512960:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078433.363832:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513088:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078433.363847:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078433.363854:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078433/real 1769078433] req@ffff9b03482c5180 x1855007617513088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078433.363862:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617513088/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078433.363874:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078433.363876:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078433.363878:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513088:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078433.363897:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513216:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078433.363908:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078433.363914:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078433/real 1769078433] req@ffff9b03482c4a80 x1855007617513216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078433.363921:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617513216/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078433.363926:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078433.363928:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078433.363929:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513216:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078433.363933:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513344:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078433.363942:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078433.363946:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078433/real 1769078433] req@ffff9b03482c5500 x1855007617513344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078433.363953:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617513344/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078433.363961:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078433.363963:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078433.363965:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513344:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078433.363968:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513472:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078433.363976:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078433.363980:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078433/real 1769078433] req@ffff9b03482c6300 x1855007617513472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078433.363986:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617513472/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078433.363990:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078433.363992:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078433.363994:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513472:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078433.875381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078434.259372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078434.259422:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078434.259443:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078434.899370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078435.283344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078435.283344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078435.283469:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078435.923354:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769078436.308110:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078436.308860:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078436.308869:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078436.947368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769078437.331403:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078437.331576:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078437.331725:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078437.971377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078438.355439:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078438.360679:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078438.365628:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078438.483869:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078438.483878:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078438.483882:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078438.483891:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5427 00000100:00080000:3.0:1769078438.483900:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078438.483907:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078438.483942:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617513600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078438.484021:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617513600:192.168.204.160@tcp:250:kworker.0 00000800:00000100:1.0:1769078438.484075:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078438.484106:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078438.484110:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078438.484112:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078438.484117:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5427 00000100:00080000:3.0:1769078438.484122:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078438.484125:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078438.484134:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078438.484148:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617513728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078438.485273:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078438/real 1769078438] req@ffff9b0352a1b480 x1855007617513600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078438.485293:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617513600/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078438.485304:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078438.485308:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078438.485312:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617513600:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078438.485335:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513728:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078438.485359:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078438.485370:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078438/real 1769078438] req@ffff9b0352a1ad80 x1855007617513728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078438.485381:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617513728/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078438.485389:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078438.485392:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078438.485395:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513728:192.168.204.160@tcp:38:kworker.0 00000100:00080000:3.0:1769078438.485477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078438.485480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078438.485483:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078438.485487:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5427 00000100:00080000:3.0:1769078438.485491:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078438.485495:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078438.485501:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078438.485513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617513856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078438.485523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078438.485541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078438.485543:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078438.485546:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5427 00000100:00080000:3.0:1769078438.485550:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078438.485554:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078438.485559:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078438.485569:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617513984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078438.485578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078438.485580:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078438.485582:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078438.485586:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5427 00000100:00080000:3.0:1769078438.485588:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078438.485592:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078438.485596:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078438.485605:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617514112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1769078438.485673:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513856:192.168.204.160@tcp:38:kworker.0 00000800:00000100:1.0:1769078438.485696:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078438.485706:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078438/real 1769078438] req@ffff9b0352a1b100 x1855007617513856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078438.485717:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617513856/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078438.485725:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078438.485727:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078438.485730:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513856:192.168.204.160@tcp:38:kworker.0 00000100:00100000:1.0:1769078438.485736:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513984:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078438.485754:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078438.485763:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078438/real 1769078438] req@ffff9b0352a18a80 x1855007617513984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078438.485773:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617513984/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078438.485780:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078438.485783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078438.485785:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617513984:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078438.485791:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617514112:192.168.204.160@tcp:8:kworker.0 00000800:00000100:1.0:1769078438.485805:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:1.0:1769078438.485814:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078438/real 1769078438] req@ffff9b0352a1a680 x1855007617514112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078438.485824:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617514112/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078438.485830:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1769078438.485833:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:1.0:1769078438.485836:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617514112:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078438.995364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078439.379389:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078439.379723:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078439.380120:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078440.019528:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078440.403388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078440.403388:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078440.403774:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078441.043359:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078441.427352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078441.428823:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078441.429034:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078442.067404:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078442.451356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078442.451356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078442.451551:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078443.091558:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078443.475351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078443.475351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078443.475413:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078443.603397:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078443.603408:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078443.603411:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078443.603418:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5432 00000100:00080000:1.0:1769078443.603425:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078443.603430:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078443.603472:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617514240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078443.603497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078443.603499:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078443.603503:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078443.603506:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5432 00000100:00080000:1.0:1769078443.603508:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078443.603511:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078443.603517:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078443.603525:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617514368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078443.603532:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078443.603534:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078443.603535:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078443.603539:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617514240:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078443.603540:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5432 00000100:00080000:1.0:1769078443.603545:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078443.603548:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078443.603552:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078443.603565:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617514496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078443.603572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078443.603575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078443.603577:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078443.603581:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5432 00000100:00080000:1.0:1769078443.603587:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078443.603590:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078443.603595:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078443.603606:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617514624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078443.603614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078443.603616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078443.603617:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078443.603621:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5432 00000100:00100000:2.0:1769078443.603625:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617514368:192.168.204.160@tcp:38:kworker.0 00000100:00080000:1.0:1769078443.603627:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078443.603634:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078443.603639:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078443.603652:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617514752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078443.603667:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617514496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078443.603693:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617514624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078443.603714:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617514752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078444.115319:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078444.499351:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078444.499509:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078444.499514:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078445.139300:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078445.523349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078445.523356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078445.523367:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078446.163340:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078446.547347:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078446.547459:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078446.549029:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000400:00000100:3.0:1769078446.675763:0:2395:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:3.0:1769078446.675817:0:2395:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1769078446.675833:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769078446.675880:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769078446.675888:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769078446.675896:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:3.0:1769078446.675902:0:2395:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:2.0:1769078446.677224:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078443/real 1769078446] req@ffff9b0343dec000 x1855007617514752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078446.677330:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617514752/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078446.677344:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078446.677350:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078446.677354:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617514752:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078446.677363:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078443/real 1769078446] req@ffff9b0343dec700 x1855007617514368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078446.677376:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617514368/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078446.677384:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078446.677396:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078446.677400:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617514368:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078446.677406:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078443/real 1769078446] req@ffff9b0343ded180 x1855007617514496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078446.677417:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617514496/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078446.677425:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078446.677428:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078446.677431:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617514496:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078446.677438:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078443/real 1769078446] req@ffff9b0343dec380 x1855007617514624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078446.677450:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617514624/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078446.677457:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078446.677461:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078446.677464:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617514624:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078446.677471:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078443/real 1769078446] req@ffff9b0343dee300 x1855007617514240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078446.677480:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617514240/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078446.677486:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078446.677489:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078446.677491:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617514240:192.168.204.160@tcp:250:kworker.0 00000100:00100000:2.0:1769078447.187330:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769078447.571650:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078447.571785:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078447.575493:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078448.211431:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078448.595336:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078448.597501:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078448.597554:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:1.0:1769078448.723450:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078448.723462:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078448.723465:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078448.723475:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5437 00000100:00080000:1.0:1769078448.723483:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078448.723490:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078448.723524:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617514880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078448.723552:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078448.723556:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078448.723558:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078448.723562:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5437 00000100:00080000:1.0:1769078448.723566:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078448.723570:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078448.723580:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078448.723594:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617514880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078448.723595:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617515008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078448.723611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078448.723614:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078448.723616:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078448.723621:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5437 00000100:00080000:1.0:1769078448.723625:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078448.723628:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078448.723633:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078448.723645:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617515136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078448.723654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078448.723656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1769078448.723664:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078448.723673:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078448.723679:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5437 00000100:00080000:1.0:1769078448.723684:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078448.723688:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078448.723693:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078448.723704:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078448/real 1769078448] req@ffff9b0343ded880 x1855007617514880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1769078448.723707:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617515264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078448.723730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078448.723733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078448.723738:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1769078448.723738:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617514880/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078448.723746:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5437 00000100:00080000:1.0:1769078448.723753:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078448.723759:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00080000:2.0:1769078448.723762:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1769078448.723764:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1769078448.723772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078448.723777:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617514880:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078448.723786:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617515392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1769078448.723814:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515008:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078448.723839:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078448.723851:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078448/real 1769078448] req@ffff9b0343def800 x1855007617515008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078448.723864:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617515008/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078448.723872:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078448.723881:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078448.723884:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515008:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078448.723890:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515136:192.168.204.160@tcp:38:kworker.0 00000800:00000100:2.0:1769078448.723914:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078448.723929:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078448/real 1769078448] req@ffff9b0343deea00 x1855007617515136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078448.723939:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617515136/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078448.723955:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078448.723962:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078448.723965:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515136:192.168.204.160@tcp:38:kworker.0 00000100:00100000:2.0:1769078448.723971:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515264:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078448.723990:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078448.724001:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078448/real 1769078448] req@ffff9b0343def100 x1855007617515264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078448.724014:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617515264/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078448.724021:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078448.724024:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078448.724026:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515264:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078448.724033:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515392:192.168.204.160@tcp:8:kworker.0 00000800:00000100:2.0:1769078448.724052:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:2.0:1769078448.724062:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078448/real 1769078448] req@ffff9b0343dedf80 x1855007617515392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1769078448.724072:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617515392/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1769078448.724079:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1769078448.724083:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:2.0:1769078448.724086:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515392:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078449.235381:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769078449.619396:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078449.621767:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078449.621894:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078450.259364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078450.643354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078450.643358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078450.643394:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078451.283353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078451.667287:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078451.667328:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078451.667348:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078452.307377:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078452.691377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078452.691474:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078452.691783:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078453.331374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078453.715439:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078453.715596:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078453.716084:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078453.843378:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078453.843389:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078453.843393:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078453.843402:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5443 00000100:00080000:1.0:1769078453.843413:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078453.843420:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078453.843454:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617515520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078453.843484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078453.843488:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078453.843490:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078453.843494:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5443 00000100:00080000:1.0:1769078453.843497:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078453.843502:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078453.843512:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078453.843524:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617515648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078453.843535:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617515520:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078453.843540:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078453.843551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078453.843553:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078453.843558:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5443 00000100:00080000:1.0:1769078453.843561:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078453.843566:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078453.843571:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078453.843587:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617515776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078453.843595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078453.843599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078453.843601:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078453.843607:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5443 00000100:00080000:1.0:1769078453.843611:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078453.843614:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078453.843621:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078453.843634:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617515904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1769078453.843646:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078453.843649:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078453.843653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078453.843655:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078453.843660:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5443 00000100:00080000:1.0:1769078453.843665:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078453.843669:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078453.843705:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078453/real 1769078453] req@ffff9b0343deca80 x1855007617515520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078453.843728:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617515520/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078453.843739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078453.843745:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000008:00100000:1.0:1769078453.843746:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078453.843751:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617515520:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078453.843766:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617516032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078453.843793:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515648:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078453.843823:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078453.843836:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078453/real 1769078453] req@ffff9b0343deed80 x1855007617515648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078453.843849:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617515648/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078453.843857:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078453.843861:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078453.843864:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515648:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078453.843873:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515776:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078453.843892:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078453.843903:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078453/real 1769078453] req@ffff9b0343ded500 x1855007617515776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078453.843915:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617515776/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078453.843923:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078453.843925:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078453.843928:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515776:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078453.843934:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515904:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078453.843953:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078453.843962:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078453/real 1769078453] req@ffff9b0343def480 x1855007617515904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078453.843972:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617515904/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078453.843980:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078453.843983:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078453.843986:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617515904:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078453.843992:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516032:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078453.844009:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078453.844018:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078453/real 1769078453] req@ffff9b0343dee680 x1855007617516032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078453.844030:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617516032/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078453.844038:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078453.844041:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078453.844044:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516032:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078454.355370:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078454.739375:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078454.739426:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078454.739429:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078455.379374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078455.763380:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078455.763442:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078455.763912:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078456.403425:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078456.787338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078456.787338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078456.787517:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078457.427334:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078457.811345:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078457.811366:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078457.812722:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078458.451371:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078458.835338:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078458.835338:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078458.835504:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078458.963355:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078458.963365:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078458.963369:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078458.963378:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5448 00000100:00080000:2.0:1769078458.963387:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078458.963393:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078458.963428:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617516160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078458.963458:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078458.963462:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078458.963464:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078458.963467:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5448 00000100:00080000:2.0:1769078458.963470:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078458.963473:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078458.963480:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078458.963489:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617516288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078458.963504:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078458.963507:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078458.963509:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078458.963513:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5448 00000100:00080000:2.0:1769078458.963516:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078458.963519:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078458.963524:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078458.963537:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617516416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078458.963547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078458.963551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078458.963554:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1769078458.963557:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617516160:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769078458.963561:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5448 00000100:00080000:2.0:1769078458.963567:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078458.963571:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078458.963577:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078458.963588:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617516544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078458.963597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078458.963604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078458.963605:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078458.963619:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5448 00000100:00080000:2.0:1769078458.963625:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1769078458.963626:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769078458.963630:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078458.963636:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078458.963648:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617516672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078458.963670:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078458/real 1769078458] req@ffff9b03482c6d80 x1855007617516160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078458.963691:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617516160/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078458.963701:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078458.963706:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078458.963710:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617516160:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078458.963717:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516288:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078458.963740:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078458.964297:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078458/real 1769078458] req@ffff9b03482c6680 x1855007617516288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078458.964312:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617516288/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078458.964321:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078458.964324:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078458.964328:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516288:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078458.964365:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516416:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078458.964387:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078458.964398:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078458/real 1769078458] req@ffff9b03482c7100 x1855007617516416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078458.964410:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617516416/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078458.964417:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078458.964420:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078458.964423:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516416:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078458.964428:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516544:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078458.964445:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078458.964454:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078458/real 1769078458] req@ffff9b03482c4000 x1855007617516544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078458.964466:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617516544/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078458.964609:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078458.964614:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078458.964617:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516544:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078458.964624:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516672:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078458.964646:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078458.964657:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078458/real 1769078458] req@ffff9b03482c6a00 x1855007617516672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078458.964669:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617516672/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078458.964677:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078458.964680:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078458.964683:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516672:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078459.475364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078459.859671:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078459.859675:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078459.859824:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078460.499392:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078460.883342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078460.883342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078460.883405:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078461.523351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078461.907358:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078461.907431:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078461.907584:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078462.547379:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769078462.931398:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078462.932090:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078462.932530:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078463.576170:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078463.955344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078463.955346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078463.955398:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078464.087346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078464.087357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078464.087362:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078464.087372:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5453 00000100:00080000:3.0:1769078464.087383:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078464.087391:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078464.087425:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617516800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078464.087454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078464.087458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078464.087460:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078464.087465:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5453 00000100:00080000:3.0:1769078464.087470:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078464.087474:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078464.087484:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078464.087499:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617516928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078464.087500:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617516800:192.168.204.160@tcp:250:kworker.0 00000100:00080000:3.0:1769078464.087517:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078464.087521:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078464.087523:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078464.087528:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5453 00000100:00080000:3.0:1769078464.087532:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078464.087537:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078464.087542:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078464.087554:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617517056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078464.087565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1769078464.087570:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:3.0:1769078464.087571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078464.087575:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078464.087580:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5453 00000100:00080000:3.0:1769078464.087585:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078464.087589:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078464.087594:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078464.087604:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617517184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078464.087615:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078464/real 1769078464] req@ffff9b0352a1a300 x1855007617516800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078464.087615:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078464.087622:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078464.087626:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078464.087633:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5453 00000100:00080000:3.0:1769078464.087638:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078464.087643:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617516800/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078464.087644:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078464.087651:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1769078464.087662:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1769078464.087667:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617517312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078464.087669:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078464.087677:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617516800:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078464.087707:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516928:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078464.087731:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078464.087743:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078464/real 1769078464] req@ffff9b0352a18700 x1855007617516928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078464.087757:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617516928/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078464.087765:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078464.087769:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078464.087773:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617516928:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078464.087780:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517056:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078464.087800:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078464.087810:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078464/real 1769078464] req@ffff9b0352a18e00 x1855007617517056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078464.087825:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617517056/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078464.087832:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078464.087836:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078464.087839:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517056:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078464.087845:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517184:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078464.087865:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078464.087877:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078464/real 1769078464] req@ffff9b0352a19f80 x1855007617517184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078464.087890:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617517184/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078464.087899:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078464.087903:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078464.087907:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517184:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078464.087913:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517312:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078464.087937:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078464.087952:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078464/real 1769078464] req@ffff9b0352a1aa00 x1855007617517312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078464.087964:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617517312/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078464.087973:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078464.087976:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078464.087979:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517312:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078464.595449:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:0.0:1769078464.979357:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078464.979387:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078464.979390:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078465.619350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078466.003354:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078466.004147:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078466.004318:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078466.643585:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078467.027377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078467.028754:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078467.029262:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078467.667451:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078468.051741:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078468.052263:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078468.052307:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078468.694035:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769078469.075472:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078469.076911:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078469.077349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:1.0:1769078469.205521:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078469.205538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078469.205543:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078469.205556:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5458 00000100:00080000:1.0:1769078469.205571:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078469.205580:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078469.205717:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dece00 x1855007617517440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078469.205820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078469.205825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078469.205827:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078469.205832:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5458 00000100:00080000:1.0:1769078469.205837:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078469.205841:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078469.205852:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078469.205868:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343defb80 x1855007617517568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078469.205879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078469.205882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078469.205884:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078469.205887:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5458 00000100:00080000:1.0:1769078469.205891:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078469.205894:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078469.205899:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078469.205908:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee680 x1855007617517696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078469.205917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078469.205920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078469.205921:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078469.205925:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5458 00000100:00080000:1.0:1769078469.205928:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078469.205932:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078469.205937:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078469.205946:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def480 x1855007617517824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078469.205960:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078469.205963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078469.205965:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078469.205968:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5458 00000100:00080000:1.0:1769078469.205972:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078469.205975:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078469.205980:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078469.205990:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded500 x1855007617517952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078469.206463:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617517440:192.168.204.160@tcp:250:kworker.0 00000800:00000100:0.0:1769078469.206545:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078469.206684:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078469/real 1769078469] req@ffff9b0343dece00 x1855007617517440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078469.206703:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dece00 x1855007617517440/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078469.206713:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078469.206717:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078469.206721:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dece00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617517440:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078469.206730:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517568:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078469.206768:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078469.206781:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078469/real 1769078469] req@ffff9b0343defb80 x1855007617517568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078469.206793:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343defb80 x1855007617517568/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078469.206800:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078469.206803:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078469.206806:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343defb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517568:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078469.206812:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517696:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078469.206831:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078469.206840:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078469/real 1769078469] req@ffff9b0343dee680 x1855007617517696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078469.206850:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee680 x1855007617517696/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078469.206857:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078469.206860:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078469.206863:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517696:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078469.206868:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517824:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078469.206885:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078469.206895:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078469/real 1769078469] req@ffff9b0343def480 x1855007617517824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078469.206910:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def480 x1855007617517824/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078469.206918:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078469.206921:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078469.206924:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517824:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078469.206930:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517952:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078469.206949:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078469.206959:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078469/real 1769078469] req@ffff9b0343ded500 x1855007617517952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078469.206970:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded500 x1855007617517952/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078469.206977:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078469.206980:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078469.206983:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617517952:192.168.204.160@tcp:8:kworker.0 00000100:00100000:2.0:1769078469.715401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078470.099430:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078470.099433:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078470.103487:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078470.739453:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078471.123360:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078471.123725:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078471.123780:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078471.763356:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078472.147351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078472.147364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078472.147459:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078472.787345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078473.171352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078473.171364:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:0.0:1769078473.171392:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078473.811344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078474.195342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078474.195342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078474.195448:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078474.323312:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078474.323322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078474.323326:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078474.323335:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5463 00000100:00080000:1.0:1769078474.323344:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078474.323350:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078474.323383:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deed80 x1855007617518080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078474.323417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078474.323421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078474.323423:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078474.323428:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5463 00000100:00080000:1.0:1769078474.323432:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078474.323437:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078474.323442:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617518080:192.168.204.160@tcp:250:kworker.0 00000008:00100000:1.0:1769078474.323449:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078474.323464:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deca80 x1855007617518208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078474.323475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078474.323478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078474.323480:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078474.323484:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5463 00000100:00080000:1.0:1769078474.323488:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078474.323492:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000800:00000100:0.0:1769078474.323494:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000008:00100000:1.0:1769078474.323499:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078474.323513:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617518336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078474.323523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078474.323527:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078474/real 1769078474] req@ffff9b0343deed80 x1855007617518080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078474.323527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078474.323532:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078474.323540:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5463 00000100:00080000:1.0:1769078474.323547:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078474.323552:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deed80 x1855007617518080/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078474.323553:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078474.323562:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1769078474.323564:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078474.323569:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078474.323572:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617518080:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078474.323577:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617518464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078474.323588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078474.323591:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518208:192.168.204.160@tcp:38:kworker.0 00000100:00080000:1.0:1769078474.323594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078474.323598:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078474.323603:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5463 00000100:00080000:1.0:1769078474.323608:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1769078474.323609:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078474.323616:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078474.323620:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078474/real 1769078474] req@ffff9b0343deca80 x1855007617518208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1769078474.323622:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078474.323633:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deca80 x1855007617518208/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078474.323640:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1769078474.323640:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617518592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078474.323646:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078474.323649:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518208:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078474.323653:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518336:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078474.323664:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078474.323672:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078474/real 1769078474] req@ffff9b0343dedf80 x1855007617518336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078474.323686:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617518336/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078474.323695:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078474.323697:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078474.323699:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518336:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078474.323703:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518464:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078474.323720:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078474.323729:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078474/real 1769078474] req@ffff9b0343def100 x1855007617518464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078474.323740:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617518464/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078474.323748:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078474.323751:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078474.323753:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518464:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078474.323766:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518592:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078474.323778:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078474.323784:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078474/real 1769078474] req@ffff9b0343deea00 x1855007617518592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078474.323791:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617518592/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078474.323795:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078474.323797:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078474.323799:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518592:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078474.835348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078475.219362:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:0.0:1769078475.219407:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078475.220471:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078475.859341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078476.243343:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078476.243343:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078476.243411:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078476.883337:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078477.267377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078477.267447:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078477.267492:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078477.907397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078478.291356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078478.291366:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078478.291430:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078478.931401:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078479.315356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078479.315498:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078479.315538:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078479.443365:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078479.443374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078479.443378:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078479.443386:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5468 00000100:00080000:2.0:1769078479.443395:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078479.443400:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078479.443434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617518720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078479.443460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078479.443463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078479.443468:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078479.443471:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5468 00000100:00080000:2.0:1769078479.443474:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078479.443476:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078479.443485:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078479.443495:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617518848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078479.443502:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078479.443504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078479.443506:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1769078479.443510:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617518720:192.168.204.160@tcp:250:kworker.0 00000100:00080000:2.0:1769078479.443510:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5468 00000100:00080000:2.0:1769078479.443515:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078479.443520:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078479.443523:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078479.443533:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617518976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078479.443538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078479.443541:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078479.443542:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078479.443544:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5468 00000100:00080000:2.0:1769078479.443547:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078479.443549:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078479.443552:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078479.443558:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617519104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078479.443563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078479.443565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078479.443567:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078479.443569:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5468 00000100:00080000:2.0:1769078479.443571:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078479.443573:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078479.443575:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078479.443580:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617519232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078479.443587:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078479.443626:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078479/real 1769078479] req@ffff9b03482c4700 x1855007617518720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078479.443644:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617518720/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078479.443651:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078479.443654:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078479.443657:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617518720:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078479.443681:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518848:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078479.443703:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078479.443714:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078479/real 1769078479] req@ffff9b03482c4e00 x1855007617518848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078479.443725:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617518848/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078479.443731:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078479.443733:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078479.443735:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518848:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078479.443740:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518976:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078479.443753:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078479.443759:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078479/real 1769078479] req@ffff9b03482c7b80 x1855007617518976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078479.443766:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617518976/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078479.443771:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078479.443772:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078479.443775:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617518976:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078479.443778:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519104:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078479.443786:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078479.443791:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078479/real 1769078479] req@ffff9b03482c5f80 x1855007617519104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078479.443798:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617519104/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078479.443802:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078479.443804:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078479.443813:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519104:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078479.443819:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519232:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078479.443856:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078479.443867:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078479/real 1769078479] req@ffff9b03482c5c00 x1855007617519232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078479.443878:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617519232/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078479.443885:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078479.443888:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078479.443891:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519232:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078479.955369:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078480.339403:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078480.339728:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078480.339779:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078480.979395:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078481.363401:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078481.363402:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078481.363525:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078482.003352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078482.387371:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078482.387963:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078482.387995:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078483.028863:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078483.411808:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078483.411811:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078483.411817:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078484.052329:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078484.435356:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:2.0:1769078484.435356:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078484.435417:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078484.563405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078484.563416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078484.563420:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078484.563430:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5473 00000100:00080000:3.0:1769078484.563438:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078484.563445:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078484.563477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1aa00 x1855007617519360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078484.563501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078484.563503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078484.563505:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078484.563508:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5473 00000100:00080000:3.0:1769078484.563510:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078484.563513:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078484.563520:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078484.563528:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19f80 x1855007617519488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078484.563534:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078484.563536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078484.563537:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078484.563539:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5473 00000100:00080000:3.0:1769078484.563541:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078484.563542:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078484.563545:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078484.563550:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18e00 x1855007617519616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078484.563556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078484.563558:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078484.563559:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078484.563561:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5473 00000100:00080000:3.0:1769078484.563562:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078484.563564:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078484.563567:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078484.563571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18700 x1855007617519744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078484.563576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078484.563577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078484.563578:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078484.563580:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5473 00000100:00080000:3.0:1769078484.563582:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078484.563583:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078484.563585:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078484.563590:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a300 x1855007617519872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078484.563704:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617519360:192.168.204.160@tcp:250:kworker.0 00000800:00000100:0.0:1769078484.563773:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078484.563804:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078484/real 1769078484] req@ffff9b0352a1aa00 x1855007617519360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078484.563816:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1aa00 x1855007617519360/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078484.563823:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078484.563826:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078484.563828:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617519360:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078484.563833:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519488:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078484.563847:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078484.563854:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078484/real 1769078484] req@ffff9b0352a19f80 x1855007617519488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078484.563865:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19f80 x1855007617519488/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078484.563869:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078484.563871:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078484.563873:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519488:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078484.563876:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519616:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078484.563884:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078484.563889:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078484/real 1769078484] req@ffff9b0352a18e00 x1855007617519616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078484.563896:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18e00 x1855007617519616/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078484.563900:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078484.563901:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078484.563903:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519616:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078484.563906:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519744:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078484.563914:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078484.563919:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078484/real 1769078484] req@ffff9b0352a18700 x1855007617519744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078484.563925:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18700 x1855007617519744/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078484.563933:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078484.563935:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078484.563937:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519744:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078484.563939:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519872:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078484.563947:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078484.563951:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078484/real 1769078484] req@ffff9b0352a1a300 x1855007617519872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078484.563957:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a300 x1855007617519872/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078484.563961:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078484.563962:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078484.563964:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617519872:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078485.075352:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078485.460934:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078485.461446:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078485.461457:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078486.099588:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769078486.483439:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1769078486.483842:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078486.483903:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078487.123343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078487.507344:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078487.507347:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078487.508534:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078488.147346:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078488.531426:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078488.533854:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078488.534385:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078489.171363:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078489.555333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078489.555349:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078489.555715:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078489.683719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078489.683729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078489.683733:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078489.683742:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5478 00000100:00080000:2.0:1769078489.683751:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078489.683758:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078489.683792:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5880 x1855007617520000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078489.683859:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617520000:192.168.204.160@tcp:250:kworker.0 00000800:00000100:0.0:1769078489.683915:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769078489.683946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078489.683950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078489.683952:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078489.683956:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5478 00000100:00080000:2.0:1769078489.683960:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078489.683964:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078489.683973:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078489.683986:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5c00 x1855007617520128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078489.684038:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078489/real 1769078489] req@ffff9b03482c5880 x1855007617520000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078489.684055:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5880 x1855007617520000/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078489.684065:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078489.684069:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078489.684072:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617520000:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078489.684091:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520128:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078489.684110:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078489.684120:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078489/real 1769078489] req@ffff9b03482c5c00 x1855007617520128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078489.684131:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5c00 x1855007617520128/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078489.684138:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078489.684141:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078489.684144:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520128:192.168.204.160@tcp:38:kworker.0 00000100:00080000:2.0:1769078489.684769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078489.684773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078489.684775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078489.684779:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5478 00000100:00080000:2.0:1769078489.684782:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078489.684785:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078489.684790:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078489.684801:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5f80 x1855007617520256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078489.684809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078489.684811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078489.684812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078489.684815:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5478 00000100:00080000:2.0:1769078489.684817:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078489.684818:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078489.684821:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078489.684826:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7b80 x1855007617520384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078489.684831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078489.684833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078489.684834:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078489.684836:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5478 00000100:00080000:2.0:1769078489.684838:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078489.684840:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078489.684842:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078489.684847:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4e00 x1855007617520512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078489.684898:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520256:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078489.684923:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078489.684934:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078489/real 1769078489] req@ffff9b03482c5f80 x1855007617520256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078489.684945:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5f80 x1855007617520256/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078489.684954:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078489.684957:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078489.684960:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520256:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078489.684966:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520384:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078489.684981:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078489.684989:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078489/real 1769078489] req@ffff9b03482c7b80 x1855007617520384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078489.685000:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7b80 x1855007617520384/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078489.685007:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078489.685009:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078489.685012:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520384:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078489.685018:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520512:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078489.685030:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078489.685038:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078489/real 1769078489] req@ffff9b03482c4e00 x1855007617520512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078489.685048:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4e00 x1855007617520512/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078489.685055:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078489.685058:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078489.685061:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520512:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078490.195341:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078490.579325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078490.579329:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078490.579445:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078491.219361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769078491.603540:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078491.603609:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078491.604482:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078492.243445:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769078492.627969:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078492.628312:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078492.630525:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078493.267365:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:0.0:1769078493.651476:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078493.651525:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078493.653469:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078494.291425:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078494.675448:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:0.0:1769078494.675453:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078494.675551:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078494.804349:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078494.804358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078494.804363:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078494.804371:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5483 00000100:00080000:3.0:1769078494.804380:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078494.804386:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078494.804477:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1a680 x1855007617520640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078494.804509:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078494.804512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078494.804514:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078494.804517:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5483 00000100:00080000:3.0:1769078494.804521:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078494.804525:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078494.804532:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078494.804541:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18a80 x1855007617520768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078494.804549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078494.804551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078494.804553:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078494.804555:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5483 00000100:00080000:3.0:1769078494.804557:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078494.804561:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078494.804564:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078494.804571:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b100 x1855007617520896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078494.804587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078494.804590:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078494.804591:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078494.804596:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5483 00000100:00080000:3.0:1769078494.804599:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078494.804603:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078494.804607:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078494.804614:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1ad80 x1855007617521024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078494.804619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078494.804621:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078494.804622:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078494.804624:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5483 00000100:00080000:3.0:1769078494.804626:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078494.804628:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078494.804630:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078494.804635:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b480 x1855007617521152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078494.808706:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617520640:192.168.204.160@tcp:250:kworker.0 00000800:00000100:0.0:1769078494.808814:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078494.808880:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078494/real 1769078494] req@ffff9b0352a1a680 x1855007617520640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078494.808902:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1a680 x1855007617520640/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078494.808916:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078494.808922:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078494.808927:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617520640:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078494.808936:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520768:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078494.808959:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078494.808970:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078494/real 1769078494] req@ffff9b0352a18a80 x1855007617520768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078494.808983:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18a80 x1855007617520768/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078494.808992:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078494.808995:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078494.808999:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520768:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078494.809006:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520896:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078494.809024:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078494.809034:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078494/real 1769078494] req@ffff9b0352a1b100 x1855007617520896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078494.809045:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b100 x1855007617520896/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078494.809059:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078494.809063:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078494.809067:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617520896:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078494.809074:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521024:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078494.809093:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078494.809102:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078494/real 1769078494] req@ffff9b0352a1ad80 x1855007617521024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078494.809113:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1ad80 x1855007617521024/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078494.809121:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078494.809124:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078494.809127:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521024:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078494.809134:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521152:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078494.809154:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078494.809163:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078494/real 1769078494] req@ffff9b0352a1b480 x1855007617521152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078494.809174:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b480 x1855007617521152/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078494.809182:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078494.809185:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078494.809188:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521152:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078495.315402:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078495.699399:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078495.699450:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078495.699778:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078496.339373:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078496.723637:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078496.723712:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078496.725626:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078497.363414:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078497.747780:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:0.0:1769078497.748228:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078497.748418:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078498.387454:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078498.771432:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078498.774036:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078498.774118:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078499.411374:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078499.795334:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078499.795505:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078499.795691:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769078499.925732:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078499.925752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078499.925757:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078499.925770:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5489 00000100:00080000:2.0:1769078499.925782:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078499.925791:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078499.925844:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4700 x1855007617521280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078499.926272:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078499.926277:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078499.926279:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078499.926284:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5489 00000100:00080000:2.0:1769078499.926290:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078499.926295:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078499.926307:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078499.926322:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617521408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078499.926334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078499.926338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078499.926340:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078499.926345:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5489 00000100:00080000:2.0:1769078499.926349:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078499.926353:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078499.926359:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078499.926370:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617521536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078499.926380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078499.926384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078499.926386:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078499.926390:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5489 00000100:00080000:2.0:1769078499.926393:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078499.926397:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078499.926402:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078499.926412:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617521664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078499.926422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078499.926425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078499.926427:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078499.926558:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5489 00000100:00080000:2.0:1769078499.926563:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078499.926568:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078499.926573:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078499.926586:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617521792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078499.926899:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617521280:192.168.204.160@tcp:250:kworker.0 00000800:00000100:0.0:1769078499.927008:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078499.927057:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078499/real 1769078499] req@ffff9b03482c4700 x1855007617521280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078499.927075:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4700 x1855007617521280/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078499.927096:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078499.927106:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078499.927111:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617521280:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078499.927119:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521408:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078499.927145:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078499.927159:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078499/real 1769078499] req@ffff9b03482c6a00 x1855007617521408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078499.927171:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617521408/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078499.927180:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078499.927184:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078499.927187:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521408:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078499.927194:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521536:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078499.927213:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078499.927224:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078499/real 1769078499] req@ffff9b03482c4000 x1855007617521536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078499.927255:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617521536/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078499.927263:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078499.927266:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078499.927269:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521536:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078499.927275:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521664:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078499.927296:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078499.927307:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078499/real 1769078499] req@ffff9b03482c7100 x1855007617521664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078499.927318:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617521664/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078499.927326:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078499.927329:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078499.927333:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521664:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078499.927338:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521792:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078499.927358:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078499.927368:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078499/real 1769078499] req@ffff9b03482c6680 x1855007617521792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078499.927387:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617521792/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078499.927396:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078499.927400:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078499.927404:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617521792:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078500.435394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078500.819382:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078500.819546:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078500.822779:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078501.459505:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078501.843366:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078501.843791:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078501.845614:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078502.484024:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769078502.867505:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078502.867516:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078502.868812:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078503.507393:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078503.891457:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078503.892715:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078503.892754:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078504.531429:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078504.915400:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078504.915505:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078504.915673:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1769078505.043382:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.043397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1769078505.043402:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078505.043423:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5494 00000100:00080000:3.0:1769078505.043435:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078505.043442:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:3.0:1769078505.043482:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18000 x1855007617521920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078505.043512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.043516:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1769078505.043518:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078505.043522:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5494 00000100:00080000:3.0:1769078505.043526:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078505.043530:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078505.043539:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078505.043552:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a18380 x1855007617522048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078505.043561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.043564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1769078505.043566:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078505.043570:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5494 00000100:00080000:3.0:1769078505.043573:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078505.043577:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078505.043582:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1769078505.043591:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19c00 x1855007617522176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078505.043600:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.043602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1769078505.043604:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078505.043607:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5494 00000100:00080000:3.0:1769078505.043610:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078505.043613:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078505.043617:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1769078505.043628:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a1b800 x1855007617522304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1769078505.043638:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1769078505.043641:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1769078505.043643:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1769078505.043647:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5494 00000100:00080000:3.0:1769078505.043651:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1769078505.043654:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:3.0:1769078505.043659:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078505.043668:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617521920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078505.043678:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0352a19500 x1855007617522432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078505.045957:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078505.046023:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff9b0352a18000 x1855007617521920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078505.046047:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18000 x1855007617521920/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078505.046060:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078505.046066:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078505.046070:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617521920:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078505.046082:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522048:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078505.046137:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078505.046150:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff9b0352a18380 x1855007617522048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078505.046163:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a18380 x1855007617522048/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078505.046178:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078505.046182:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078505.046186:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a18380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522048:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078505.046194:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522176:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078505.046214:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078505.046224:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff9b0352a19c00 x1855007617522176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078505.046256:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19c00 x1855007617522176/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078505.046264:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078505.046268:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078505.046272:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522176:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078505.046278:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522304:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078505.046299:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078505.046310:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff9b0352a1b800 x1855007617522304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078505.046321:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a1b800 x1855007617522304/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078505.046328:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078505.046332:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078505.046337:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a1b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522304:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078505.046373:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522432:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078505.046392:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078505.046413:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078505/real 1769078505] req@ffff9b0352a19500 x1855007617522432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078505.046428:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0352a19500 x1855007617522432/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078505.046436:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078505.046439:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078505.046442:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0352a19500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522432:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078505.555342:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078505.939348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078505.939348:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078505.939688:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078506.579490:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:2.0:1769078506.963353:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:1.0:1769078506.963618:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078506.963673:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078507.603328:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769078507.987510:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078507.987541:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078507.987623:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078508.627362:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078509.011325:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078509.011325:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078509.011393:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078509.651310:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078510.035332:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078510.035332:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078510.035379:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1769078510.163358:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078510.163366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078510.163370:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078510.163377:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5499 00000100:00080000:1.0:1769078510.163384:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078510.163389:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078510.163417:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617522560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078510.163435:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078510.163437:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078510.163438:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078510.163441:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5499 00000100:00080000:1.0:1769078510.163443:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078510.163446:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078510.163453:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078510.163460:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617522688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078510.163469:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617522560:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078510.163469:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078510.163474:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078510.163476:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078510.163481:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5499 00000100:00080000:1.0:1769078510.163483:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078510.163485:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078510.163487:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078510.163499:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617522816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078510.163506:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078510.163508:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078510.163509:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078510.163511:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5499 00000100:00080000:1.0:1769078510.163524:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078510.163526:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078510.163529:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078510.163537:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617522944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078510.163542:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078510.163545:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078510.163546:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078510.163548:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5499 00000100:00080000:1.0:1769078510.163550:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078510.163553:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078510.163556:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078510.163563:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617523072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078510.163568:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078510.163593:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078510.163607:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078510.163619:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:1.0:1769078510.675350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078511.059355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078511.059355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:1.0:1769078511.059396:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078511.699351:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078512.083367:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078512.083424:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078512.083459:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078512.723348:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078513.107381:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078513.107484:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078513.107660:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000400:00000100:2.0:1769078513.235743:0:2396:0:(lib-socket.c:453:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.204.160:988 00000400:00000100:2.0:1769078513.235800:0:2396:0:(acceptor.c:115:lnet_connect_console_error()) Connection to 192.168.204.160@tcp at host 192.168.204.160 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1769078513.235822:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769078513.235866:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769078513.235875:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769078513.235883:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000800:00000100:2.0:1769078513.235891:0:2396:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.204.160@tcp 00000100:00100000:0.0:1769078513.236116:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078510/real 1769078513] req@ffff9b0343ded180 x1855007617523072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078513.236151:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617523072/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078513.236170:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078513.236175:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078513.236180:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523072:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078513.236188:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078510/real 1769078513] req@ffff9b0343ded880 x1855007617522688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078513.236200:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617522688/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078513.236207:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078513.236210:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078513.236214:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522688:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078513.236221:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078510/real 1769078513] req@ffff9b0343dee300 x1855007617522816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078513.236256:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617522816/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078513.236277:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078513.236284:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078513.236288:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522816:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078513.236298:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078510/real 1769078513] req@ffff9b0343dec380 x1855007617522944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078513.236322:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617522944/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078513.236331:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078513.236334:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078513.236337:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617522944:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078513.236354:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078510/real 1769078513] req@ffff9b0343def800 x1855007617522560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078513.236367:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617522560/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078513.236374:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078513.236378:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078513.236381:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617522560:192.168.204.160@tcp:250:kworker.0 00000100:00100000:1.0:1769078513.747349:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078514.131344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078514.131386:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078514.136317:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078514.771320:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078515.155357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078515.155373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078515.155421:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1769078515.283435:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078515.283455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078515.283459:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078515.283469:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5504 00000100:00080000:2.0:1769078515.283481:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078515.283487:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078515.283520:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617523200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078515.283539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078515.283542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078515.283544:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078515.283547:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5504 00000100:00080000:2.0:1769078515.283550:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078515.283555:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078515.283564:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078515.283576:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617523328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078515.283585:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078515.283588:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078515.283590:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078515.283593:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5504 00000100:00080000:2.0:1769078515.283596:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078515.283600:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078515.283604:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078515.283612:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617523456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078515.283620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078515.283623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078515.283625:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078515.283629:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5504 00000100:00080000:2.0:1769078515.283632:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078515.283635:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078515.283640:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078515.283650:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617523584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078515.283658:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078515.283661:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078515.283663:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078515.283666:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5504 00000100:00080000:2.0:1769078515.283668:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078515.283671:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078515.283674:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078515.283680:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617523712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078515.283959:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617523200:192.168.204.160@tcp:250:kworker.0 00000800:00000100:0.0:1769078515.284062:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078515.284116:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078515/real 1769078515] req@ffff9b03482c6d80 x1855007617523200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078515.284139:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617523200/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078515.284151:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078515.284156:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078515.284160:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617523200:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078515.284169:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523328:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078515.284195:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078515.284208:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078515/real 1769078515] req@ffff9b03482c6300 x1855007617523328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078515.284221:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617523328/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078515.284265:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078515.284271:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078515.284275:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523328:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078515.284281:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523456:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078515.284303:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078515.284314:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078515/real 1769078515] req@ffff9b03482c5500 x1855007617523456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078515.284325:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617523456/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078515.284333:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078515.284337:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078515.284340:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523456:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078515.284347:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523584:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078515.284366:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078515.284376:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078515/real 1769078515] req@ffff9b03482c4a80 x1855007617523584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078515.284392:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617523584/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078515.284399:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078515.284402:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078515.284405:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523584:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078515.284411:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523712:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078515.284428:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078515.284437:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078515/real 1769078515] req@ffff9b03482c5180 x1855007617523712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078515.284447:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617523712/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078515.284453:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078515.284456:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078515.284460:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523712:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078515.795450:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078516.179352:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078516.179352:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078516.179427:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078516.819344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078517.203348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078517.203373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078517.203419:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078517.843350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078518.227354:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078518.227355:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078518.228332:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078518.867344:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078519.251350:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078519.251416:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078519.252148:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078519.891343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078520.275355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:3.0:1769078520.277402:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078520.277420:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00080000:2.0:1769078520.403368:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078520.403380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078520.403385:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078520.403394:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5509 00000100:00080000:2.0:1769078520.403403:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078520.403420:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078520.403466:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4380 x1855007617523840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078520.403492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078520.403496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078520.403498:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078520.403502:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5509 00000100:00080000:2.0:1769078520.403505:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078520.403509:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078520.403518:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078520.403530:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5180 x1855007617523968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078520.403539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078520.403542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078520.403543:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078520.403546:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5509 00000100:00080000:2.0:1769078520.403549:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078520.403552:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078520.403556:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078520.403565:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4a80 x1855007617524096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078520.403582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078520.403585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078520.403587:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078520.403592:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5509 00000100:00080000:2.0:1769078520.403595:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078520.403599:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078520.403605:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617523840:192.168.204.160@tcp:250:kworker.0 00000008:00100000:2.0:1769078520.403605:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078520.403616:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c5500 x1855007617524224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078520.403624:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078520.403627:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078520.403629:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078520.403631:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5509 00000100:00080000:2.0:1769078520.403634:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078520.403636:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078520.403640:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078520.403647:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6300 x1855007617524352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1769078520.403742:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078520.403777:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078520/real 1769078520] req@ffff9b03482c4380 x1855007617523840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078520.403794:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4380 x1855007617523840/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078520.403809:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078520.403813:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078520.403817:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617523840:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078520.403823:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523968:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078520.403842:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078520.403852:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078520/real 1769078520] req@ffff9b03482c5180 x1855007617523968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078520.403863:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5180 x1855007617523968/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078520.403870:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078520.403873:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078520.403876:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617523968:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078520.403882:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524096:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078520.403897:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078520.403906:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078520/real 1769078520] req@ffff9b03482c4a80 x1855007617524096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078520.403916:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4a80 x1855007617524096/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078520.403923:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078520.403926:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078520.403929:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524096:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078520.403963:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524224:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078520.403981:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078520.403990:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078520/real 1769078520] req@ffff9b03482c5500 x1855007617524224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078520.403999:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c5500 x1855007617524224/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078520.404005:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078520.404008:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078520.404011:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524224:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078520.404016:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524352:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078520.404029:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078520.404037:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078520/real 1769078520] req@ffff9b03482c6300 x1855007617524352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078520.404047:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6300 x1855007617524352/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078520.404054:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078520.404056:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078520.404059:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524352:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078520.915347:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078521.299357:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078521.299429:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078521.299669:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078521.939615:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078522.323375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:1.0:1769078522.323450:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00010000:00010000:2.0:1769078522.323457:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078522.963394:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078523.347373:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078523.347396:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078523.347436:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078523.987503:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078524.371326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078524.371385:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078524.371467:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078525.012361:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769078525.395560:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078525.406326:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078525.406386:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:2.0:1769078525.523451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078525.523462:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1769078525.523466:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078525.523475:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5514 00000100:00080000:2.0:1769078525.523484:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078525.523490:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:2.0:1769078525.523524:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6d80 x1855007617524480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078525.523579:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617524480:192.168.204.160@tcp:250:kworker.0 00000800:00000100:0.0:1769078525.523633:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:2.0:1769078525.523661:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078525.523664:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1769078525.523666:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078525.523670:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5514 00000100:00080000:2.0:1769078525.523674:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078525.523678:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078525.523686:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078525.523697:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6680 x1855007617524608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078525.523748:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078525/real 1769078525] req@ffff9b03482c6d80 x1855007617524480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078525.523765:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6d80 x1855007617524480/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078525.523774:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078525.523778:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078525.523782:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617524480:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078525.523802:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524608:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078525.523820:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078525.523831:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078525/real 1769078525] req@ffff9b03482c6680 x1855007617524608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078525.523842:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6680 x1855007617524608/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078525.523850:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078525.523853:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078525.523857:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524608:192.168.204.160@tcp:38:kworker.0 00000100:00080000:2.0:1769078525.528208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078525.528216:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1769078525.528219:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078525.528228:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5514 00000100:00080000:2.0:1769078525.528246:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078525.528253:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078525.528263:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1769078525.528289:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c7100 x1855007617524736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078525.528303:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078525.528306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1769078525.528308:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078525.528312:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5514 00000100:00080000:2.0:1769078525.528315:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078525.528318:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078525.528323:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078525.528331:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c4000 x1855007617524864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1769078525.528340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1769078525.528343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1769078525.528344:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1769078525.528348:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5514 00000100:00080000:2.0:1769078525.528350:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1769078525.528353:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:2.0:1769078525.528356:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1769078525.528363:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b03482c6a00 x1855007617524992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078525.528451:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524736:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078525.528495:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078525.528516:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078525/real 1769078525] req@ffff9b03482c7100 x1855007617524736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078525.528532:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c7100 x1855007617524736/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078525.528548:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078525.528552:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078525.528555:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524736:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078525.528562:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524864:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078525.528576:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078525.528584:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078525/real 1769078525] req@ffff9b03482c4000 x1855007617524864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078525.528595:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c4000 x1855007617524864/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078525.528603:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078525.528606:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078525.528609:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524864:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078525.528614:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524992:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078525.528626:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078525.528634:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078525/real 1769078525] req@ffff9b03482c6a00 x1855007617524992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078525.528644:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b03482c6a00 x1855007617524992/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078525.528652:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078525.528655:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078525.528658:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b03482c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617524992:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078526.035350:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078526.419475:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078526.420519:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078526.446201:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078527.059326:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078527.443377:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078527.443466:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078527.510155:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078528.083426:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769078528.467810:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078528.467916:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078528.531333:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:1.0:1769078529.107439:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078529.491521:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078529.492554:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078529.555357:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078530.131490:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078530.515421:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078530.515665:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078530.579992:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:1.0:1769078530.644028:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078530.644044:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078530.644049:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078530.644062:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5519 00000100:00080000:1.0:1769078530.644075:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078530.644084:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078530.644132:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec700 x1855007617525120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078530.644174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078530.644178:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078530.644180:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078530.644185:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5519 00000100:00080000:1.0:1769078530.644189:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078530.644194:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078530.644204:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078530.644218:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec000 x1855007617525248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078530.644228:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1769078530.644278:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617525120:192.168.204.160@tcp:250:kworker.0 00000100:00080000:1.0:1769078530.644285:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078530.644292:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078530.644299:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5519 00000100:00080000:1.0:1769078530.644304:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078530.644311:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078530.644317:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078530.644336:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def800 x1855007617525376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078530.644347:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078530.644352:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078530.644354:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078530.644359:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5519 00000800:00000100:0.0:1769078530.644367:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:1.0:1769078530.644367:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078530.644376:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078530.644383:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078530.644397:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dec380 x1855007617525504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078530.644407:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078530.644410:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078530.644413:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078530.644418:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5519 00000100:00080000:1.0:1769078530.644422:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1769078530.644426:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078530/real 1769078530] req@ffff9b0343dec700 x1855007617525120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1769078530.644430:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078530.644438:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078530.644455:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dee300 x1855007617525632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1769078530.644466:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec700 x1855007617525120/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078530.644480:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078530.644486:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078530.644491:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617525120:192.168.204.160@tcp:250:kworker.0 00000100:00100000:0.0:1769078530.644501:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617525248:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078530.644526:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078530.644553:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078530/real 1769078530] req@ffff9b0343dec000 x1855007617525248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078530.644566:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec000 x1855007617525248/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078530.644574:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078530.644577:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078530.644581:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617525248:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078530.644605:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617525376:192.168.204.160@tcp:38:kworker.0 00000800:00000100:0.0:1769078530.644625:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078530.644636:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078530/real 1769078530] req@ffff9b0343def800 x1855007617525376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078530.644647:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def800 x1855007617525376/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078530.644654:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078530.644657:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078530.644660:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617525376:192.168.204.160@tcp:38:kworker.0 00000100:00100000:0.0:1769078530.644665:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617525504:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078530.644683:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078530.644692:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078530/real 1769078530] req@ffff9b0343dec380 x1855007617525504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078530.644713:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dec380 x1855007617525504/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078530.644721:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078530.644724:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078530.644727:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dec380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617525504:192.168.204.160@tcp:8:kworker.0 00000100:00100000:0.0:1769078530.644733:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617525632:192.168.204.160@tcp:8:kworker.0 00000800:00000100:0.0:1769078530.644752:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:0.0:1769078530.644763:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078530/real 1769078530] req@ffff9b0343dee300 x1855007617525632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1769078530.644772:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dee300 x1855007617525632/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1769078530.644780:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1769078530.644783:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:0.0:1769078530.644786:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dee300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617525632:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078531.155315:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769078531.541308:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078531.541348:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078531.603441:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078532.179507:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:2.0:1769078532.563583:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1769078532.563876:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078532.628059:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078533.205343:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078533.587351:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078533.587647:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078533.651355:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078534.227332:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:3.0:1769078534.614501:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078534.615573:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078534.675316:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078535.252106:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00010000:00010000:3.0:1769078535.636361:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1769078535.636470:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00000100:00100000:2.0:1769078535.699388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:1.0:1769078535.765266:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078535.765275:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1769078535.765279:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078535.765289:0:36:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5524 00000100:00080000:1.0:1769078535.765299:0:36:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078535.765306:0:36:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:1.0:1769078535.765335:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded880 x1855007617525760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078535.765365:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078535.765368:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1769078535.765370:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078535.765374:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5524 00000100:00080000:1.0:1769078535.765377:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078535.765380:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078535.765389:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078535.765402:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343ded180 x1855007617525888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078535.765410:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078535.765413:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1769078535.765415:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078535.765418:0:36:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5524 00000100:00080000:1.0:1769078535.765422:0:36:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078535.765425:0:36:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078535.765429:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1769078535.765440:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343deea00 x1855007617526016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078535.765447:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078535.765449:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1769078535.765451:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078535.765455:0:36:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5524 00000100:00080000:1.0:1769078535.765458:0:36:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078535.765461:0:36:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078535.765465:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078535.765472:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343def100 x1855007617526144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1769078535.765479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1769078535.765482:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1769078535.765483:0:36:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1769078535.765486:0:36:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5524 00000100:00080000:1.0:1769078535.765489:0:36:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1769078535.765492:0:36:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:1.0:1769078535.765495:0:36:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1769078535.765503:0:36:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0343dedf80 x1855007617526272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078535.765792:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617525760:192.168.204.160@tcp:250:kworker.0 00000800:00000100:3.0:1769078535.765856:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078535.766109:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078535/real 1769078535] req@ffff9b0343ded880 x1855007617525760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078535.766129:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded880 x1855007617525760/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078535.766138:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078535.766142:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078535.766146:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617525760:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078535.766153:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617525888:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078535.766179:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078535.766195:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078535/real 1769078535] req@ffff9b0343ded180 x1855007617525888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078535.766211:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343ded180 x1855007617525888/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078535.766219:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078535.766223:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078535.766228:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343ded180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617525888:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078535.766403:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526016:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078535.766433:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078535.766447:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078535/real 1769078535] req@ffff9b0343deea00 x1855007617526016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078535.766532:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343deea00 x1855007617526016/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078535.766541:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078535.766544:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078535.766548:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343deea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526016:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078535.766554:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526144:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078535.766577:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078535.766589:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078535/real 1769078535] req@ffff9b0343def100 x1855007617526144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078535.766608:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343def100 x1855007617526144/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078535.766616:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078535.766619:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078535.766622:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343def100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526144:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078535.766628:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526272:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078535.766648:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078535.766658:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078535/real 1769078535] req@ffff9b0343dedf80 x1855007617526272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078535.766669:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0343dedf80 x1855007617526272/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078535.766676:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078535.766679:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078535.766682:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0343dedf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526272:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078536.275364:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078536.659364:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078536.660963:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078536.723372:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078537.299345:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078537.683342:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078537.683425:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078537.747346:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078538.323367:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078538.707407:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078538.707425:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078538.771342:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078539.347327:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078539.731358:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078539.731428:0:60221:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078539.795344:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078540.371355:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078540.755384:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078540.755591:0:39339:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078540.819575:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00080000:0.0:1769078540.884202:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7835f93-1e5d-43a4-8110-91f072d9ae65->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078540.884216:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1769078540.884221:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078540.884252:0:8:0:(import.c:509:import_select_connection()) MGC192.168.204.160@tcp: connect to NID 192.168.204.160@tcp last attempt 5530 00000100:00080000:0.0:1769078540.884263:0:8:0:(import.c:588:import_select_connection()) MGC192.168.204.160@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078540.884271:0:8:0:(import.c:623:import_select_connection()) MGC192.168.204.160@tcp: import ffff9b0353290000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000100:00100000:0.0:1769078540.884316:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3b100 x1855007617526400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078540.884492:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617526400:192.168.204.160@tcp:250:kworker.0 00000800:00000100:3.0:1769078540.884570:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00080000:0.0:1769078540.884883:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078540.884895:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1769078540.884897:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078540.884902:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5530 00000100:00080000:0.0:1769078540.884906:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078540.884910:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff9b0342ad2800: import ffff9b0343fe4000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078540.884920:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078540.884935:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3ad80 x1855007617526528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078540.886010:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078540/real 1769078540] req@ffff9b0365a3b100 x1855007617526400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078540.886035:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3b100 x1855007617526400/t0(0) o250->MGC192.168.204.160@tcp@192.168.204.160@tcp:26/25 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078540.886045:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0353290000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078540.886050:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078540.886054:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7835f93-1e5d-43a4-8110-91f072d9ae65:2405:1855007617526400:192.168.204.160@tcp:250:kworker.0 00000100:00100000:3.0:1769078540.886082:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526528:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078540.886110:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078540.886121:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078540/real 1769078540] req@ffff9b0365a3ad80 x1855007617526528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078540.886133:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3ad80 x1855007617526528/t0(0) o38->lustre-MDT0000-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078540.886140:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0343fe4000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078540.886143:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078540.886146:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526528:192.168.204.160@tcp:38:kworker.0 00000100:00080000:0.0:1769078540.886342:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078540.886347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1769078540.886353:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078540.886358:0:8:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5530 00000100:00080000:0.0:1769078540.886363:0:8:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078540.886369:0:8:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff9b0342ad2800: import ffff9b0352ab6000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078540.886376:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681348 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1769078540.886391:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a3bb80 x1855007617526656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078540.886402:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078540.886406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1769078540.886407:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078540.886412:0:8:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5530 00000100:00080000:0.0:1769078540.886416:0:8:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078540.886419:0:8:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff9b0342ad2800: import ffff9b0352ab1000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078540.886424:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078540.886434:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39500 x1855007617526784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1769078540.886442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 249ac4ab-c06c-4166-99d5-c4c7e1cc2791->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1769078540.886445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1769078540.886447:0:8:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1769078540.886451:0:8:0:(import.c:509:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: connect to NID 192.168.204.160@tcp last attempt 5530 00000100:00080000:0.0:1769078540.886453:0:8:0:(import.c:588:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1769078540.886456:0:8:0:(import.c:623:import_select_connection()) lustre-OST0001-osc-ffff9b0342ad2800: import ffff9b0352ab7000 using connection 192.168.204.160@tcp/192.168.204.160@tcp 00000008:00100000:0.0:1769078540.886461:0:8:0:(osc_request.c:3784:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681348 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1769078540.886471:0:8:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9b0365a39180 x1855007617526912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1769078540.886600:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526656:192.168.204.160@tcp:38:kworker.0 00000800:00000100:3.0:1769078540.886636:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078540.886649:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078540/real 1769078540] req@ffff9b0365a3bb80 x1855007617526656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078540.886669:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a3bb80 x1855007617526656/t0(0) o38->lustre-MDT0001-mdc-ffff9b0342ad2800@192.168.204.160@tcp:12/10 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078540.886678:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab6000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078540.886681:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078540.886685:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a3bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526656:192.168.204.160@tcp:38:kworker.0 00000100:00100000:3.0:1769078540.886691:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526784:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078540.886711:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078540.886721:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078540/real 1769078540] req@ffff9b0365a39500 x1855007617526784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078540.886733:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39500 x1855007617526784/t0(0) o8->lustre-OST0000-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078540.886739:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab1000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078540.886743:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078540.886746:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526784:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078540.886751:0:2405:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526912:192.168.204.160@tcp:8:kworker.0 00000800:00000100:3.0:1769078540.886769:0:2405:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.204.160@tcp 00000100:00100000:3.0:1769078540.886779:0:2405:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1769078540/real 1769078540] req@ffff9b0365a39180 x1855007617526912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1769078540.886789:0:2405:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9b0365a39180 x1855007617526912/t0(0) o8->lustre-OST0001-osc-ffff9b0342ad2800@192.168.204.160@tcp:28/4 lens 520/544 e 0 to 1 dl 1769078565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1769078540.886886:0:2405:0:(import.c:65:import_set_state_nolock()) ffff9b0352ab7000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1769078540.886899:0:2405:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.204.160@tcp failed (-110) 00000100:00100000:3.0:1769078540.886903:0:2405:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff9b0365a39180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:249ac4ab-c06c-4166-99d5-c4c7e1cc2791:2405:1855007617526912:192.168.204.160@tcp:8:kworker.0 00000100:00100000:3.0:1769078541.395368:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078541.779372:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:3.0:1769078541.779411:0:39340:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078541.843377:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078542.419318:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078542.803375:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078542.803448:0:46917:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078542.867388:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078543.443397:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds 00000100:00100000:1.0:1769078543.827368:0:60112:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3e180 going to sleep for 0 seconds 00010000:00010000:2.0:1769078543.831192:0:60220:0:(ldlm_lockd.c:2796:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1769078543.891350:0:57730:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0360b61c00 going to sleep for 0 seconds 00000100:00100000:3.0:1769078544.467353:0:69591:0:(client.c:2690:ptlrpc_set_wait()) set ffff9b0352a3ee40 going to sleep for 0 seconds Debug log: 81815 lines, 81815 kept, 0 dropped, 0 bad.